move everything into experimental

docs/extras/modules/chains/how_to/learned_prompt_optimization.ipynb

@@ -17,7 +17,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Install necessary packages\n",
+    "# ! pip install langchain langchain-experimental matplotlib"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -33,14 +43,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "# pick and configure the LLM of your choice\n",
     "\n",
     "from langchain.llms import OpenAI\n",
-    "llm = OpenAI(engine=\"text-davinci-003\")\n"
+    "llm = OpenAI(engine=\"text-davinci-003\")"
    ]
   },
   {
@@ -93,7 +103,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import langchain.chains.rl_chain as rl_chain\n",
+    "import langchain_experimental.rl_chain as rl_chain\n",
     "\n",
     "chain = rl_chain.PickBest.from_llm(llm=llm, prompt=PROMPT)\n"
    ]
@@ -466,12 +476,10 @@
     }
    ],
    "source": [
-    "# note matplotlib is not a dependency of langchain so you need to install to plot\n",
-    "\n",
-    "# from matplotlib import pyplot as plt\n",
-    "# chain.metrics.to_pandas()['score'].plot(label=\"default learning policy\")\n",
-    "# random_chain.metrics.to_pandas()['score'].plot(label=\"random selection policy\")\n",
-    "# plt.legend()\n",
+    "from matplotlib import pyplot as plt\n",
+    "chain.metrics.to_pandas()['score'].plot(label=\"default learning policy\")\n",
+    "random_chain.metrics.to_pandas()['score'].plot(label=\"random selection policy\")\n",
+    "plt.legend()\n",
     "\n",
     "print(f\"The final average score for the default policy, calculated over a rolling window, is: {chain.metrics.to_pandas()['score'].iloc[-1]}\")\n",
     "print(f\"The final average score for the random policy, calculated over a rolling window, is: {random_chain.metrics.to_pandas()['score'].iloc[-1]}\")"
@@ -816,7 +824,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.16"
+   "version": "3.9.17"
   },
   "orig_nbformat": 4
  },

libs/experimental/langchain_experimental/rl_chain/__init__.py

@@ -1,6 +1,6 @@
 import logging
 
-from langchain.chains.rl_chain.base import (
+from langchain_experimental.rl_chain.base import (
     AutoSelectionScorer,
     BasedOn,
     Embed,
@@ -12,7 +12,7 @@ from langchain.chains.rl_chain.base import (
     embed,
     stringify_embedding,
 )
-from langchain.chains.rl_chain.pick_best_chain import (
+from langchain_experimental.rl_chain.pick_best_chain import (
     PickBest,
     PickBestEvent,
     PickBestFeatureEmbedder,

libs/experimental/langchain_experimental/rl_chain/base.py

@@ -19,19 +19,19 @@ from typing import (
 from langchain.callbacks.manager import CallbackManagerForChainRun
 from langchain.chains.base import Chain
 from langchain.chains.llm import LLMChain
-from langchain.chains.rl_chain.metrics import (
+from langchain_experimental.rl_chain.metrics import (
     MetricsTrackerAverage,
     MetricsTrackerRollingWindow,
 )
-from langchain.chains.rl_chain.model_repository import ModelRepository
-from langchain.chains.rl_chain.vw_logger import VwLogger
+from langchain_experimental.rl_chain.model_repository import ModelRepository
+from langchain_experimental.rl_chain.vw_logger import VwLogger
 from langchain.prompts import (
     BasePromptTemplate,
     ChatPromptTemplate,
     HumanMessagePromptTemplate,
     SystemMessagePromptTemplate,
 )
-from langchain.pydantic_v1 import BaseModel, Extra, root_validator
+from langchain_experimental.pydantic_v1 import BaseModel, Extra, root_validator
 
 if TYPE_CHECKING:
     import vowpal_wabbit_next as vw

libs/experimental/langchain_experimental/rl_chain/pick_best_chain.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import logging
 from typing import Any, Dict, List, Optional, Tuple, Type, Union
 
-import langchain.chains.rl_chain.base as base
+import langchain_experimental.rl_chain.base as base
 from langchain.base_language import BaseLanguageModel
 from langchain.callbacks.manager import CallbackManagerForChainRun
 from langchain.chains.llm import LLMChain
@@ -307,6 +307,7 @@ class PickBest(base.RLChain[PickBestEvent]):
             ]
 
         kwargs["vw_cmd"] = vw_cmd
+        logger.info(f"vw_cmd: {vw_cmd}")
 
         super().__init__(*args, **kwargs)
 

libs/experimental/pyproject.toml

@@ -14,6 +14,9 @@ langchain = ">=0.0.239"
 presidio-anonymizer = {version = "^2.2.33", optional = true}
 presidio-analyzer = {version = "^2.2.33", optional = true}
 faker = {version = "^19.3.1", optional = true}
+vowpal-wabbit-next = {version = "0.6.0", optional = true}
+sentence-transformers = {version = "^2", optional = true}
+pandas = {version = "^2.0.1", optional = true}
 
 
 [tool.poetry.group.lint.dependencies]
@@ -42,6 +45,8 @@ extended_testing = [
  "presidio-anonymizer",
  "presidio-analyzer",
  "faker",
+ "vowpal-wabbit-next",
+ "sentence-transformers",
 ]
 
 [tool.ruff]

libs/experimental/tests/unit_tests/rl_chain/test_pick_best_chain_call.py

@@ -3,8 +3,8 @@ from typing import Any, Dict
 import pytest
 from test_utils import MockEncoder, MockEncoderReturnsList
 
-import langchain.chains.rl_chain.base as rl_chain
-import langchain.chains.rl_chain.pick_best_chain as pick_best_chain
+import langchain_experimental.rl_chain.base as rl_chain
+import langchain_experimental.rl_chain.pick_best_chain as pick_best_chain
 from langchain.chat_models import FakeListChatModel
 from langchain.prompts.prompt import PromptTemplate
 
@@ -332,7 +332,7 @@ def test_default_embeddings_mixed_w_explicit_user_embeddings() -> None:
 @pytest.mark.requires("vowpal_wabbit_next", "sentence_transformers")
 def test_default_no_scorer_specified() -> None:
     _, PROMPT = setup()
-    chain_llm = FakeListChatModel(responses=[100])
+    chain_llm = FakeListChatModel(responses=["hey", "100"])
     chain = pick_best_chain.PickBest.from_llm(
         llm=chain_llm,
         prompt=PROMPT,
@@ -345,7 +345,7 @@ def test_default_no_scorer_specified() -> None:
         action=rl_chain.ToSelectFrom(["0", "1", "2"]),
     )
     # chain llm used for both basic prompt and for scoring
-    assert response["response"] == "100"
+    assert response["response"] == "hey"
     selection_metadata = response["selection_metadata"]
     assert selection_metadata.selected.score == 100.0
 
@@ -374,7 +374,7 @@ def test_explicitly_no_scorer() -> None:
 @pytest.mark.requires("vowpal_wabbit_next", "sentence_transformers")
 def test_auto_scorer_with_user_defined_llm() -> None:
     llm, PROMPT = setup()
-    scorer_llm = FakeListChatModel(responses=[300])
+    scorer_llm = FakeListChatModel(responses=["300"])
     chain = pick_best_chain.PickBest.from_llm(
         llm=llm,
         prompt=PROMPT,
@@ -418,8 +418,9 @@ def test_calling_chain_w_reserved_inputs_throws() -> None:
 
 @pytest.mark.requires("vowpal_wabbit_next", "sentence_transformers")
 def test_activate_and_deactivate_scorer() -> None:
-    llm, PROMPT = setup()
-    scorer_llm = FakeListChatModel(responses=[300])
+    _, PROMPT = setup()
+    llm = FakeListChatModel(responses=["hey1", "hey2", "hey3"])
+    scorer_llm = FakeListChatModel(responses=["300", "400"])
     chain = pick_best_chain.PickBest.from_llm(
         llm=llm,
         prompt=PROMPT,
@@ -433,7 +434,7 @@ def test_activate_and_deactivate_scorer() -> None:
         action=pick_best_chain.base.ToSelectFrom(["0", "1", "2"]),
     )
     # chain llm used for both basic prompt and for scoring
-    assert response["response"] == "hey"
+    assert response["response"] == "hey1"
     selection_metadata = response["selection_metadata"]
     assert selection_metadata.selected.score == 300.0
 
@@ -442,7 +443,7 @@ def test_activate_and_deactivate_scorer() -> None:
         User=pick_best_chain.base.BasedOn("Context"),
         action=pick_best_chain.base.ToSelectFrom(["0", "1", "2"]),
     )
-    assert response["response"] == "hey"
+    assert response["response"] == "hey2"
     selection_metadata = response["selection_metadata"]
     assert selection_metadata.selected.score is None
 
@@ -451,6 +452,6 @@ def test_activate_and_deactivate_scorer() -> None:
         User=pick_best_chain.base.BasedOn("Context"),
         action=pick_best_chain.base.ToSelectFrom(["0", "1", "2"]),
     )
-    assert response["response"] == "hey"
+    assert response["response"] == "hey3"
     selection_metadata = response["selection_metadata"]
-    assert selection_metadata.selected.score == 300.0
+    assert selection_metadata.selected.score == 400.0

libs/experimental/tests/unit_tests/rl_chain/test_pick_best_text_embedder.py

@@ -1,8 +1,8 @@
 import pytest
 from test_utils import MockEncoder
 
-import langchain.chains.rl_chain.base as rl_chain
-import langchain.chains.rl_chain.pick_best_chain as pick_best_chain
+import langchain_experimental.rl_chain.base as rl_chain
+import langchain_experimental.rl_chain.pick_best_chain as pick_best_chain
 
 encoded_keyword = "[encoded]"
 

libs/experimental/tests/unit_tests/rl_chain/test_rl_chain_base_embedder.py

@@ -3,7 +3,7 @@ from typing import List, Union
 import pytest
 from test_utils import MockEncoder
 
-import langchain.chains.rl_chain.base as base
+import langchain_experimental.rl_chain.base as base
 
 encoded_keyword = "[encoded]"
 

libs/langchain/poetry.lock

@@ -9800,33 +9800,6 @@ urllib3 = {version = "<2", markers = "python_version < \"3.10\""}
 wrapt = "*"
 yarl = "*"
 
-[[package]]
-name = "vowpal-wabbit-next"
-version = "0.6.0"
-description = "Experimental python bindings for VowpalWabbit"
-optional = true
-python-versions = ">=3.7"
-files = [
-    {file = "vowpal-wabbit-next-0.6.0.tar.gz", hash = "sha256:f0381614d99fac6a0f52e995ee0bfc7b681054f397bea7ff08b8a523d5315a54"},
-    {file = "vowpal_wabbit_next-0.6.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:cfbb831cfe9eb81185aff7cdca437ae17c6d9aca8d74e26c326e3ef4ee8e81e7"},
-    {file = "vowpal_wabbit_next-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d31829778f9c600f5c121f614516ca1bc9ede5d1bc77b1eb3b59b32d9138db9"},
-    {file = "vowpal_wabbit_next-0.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:714347606ab302a2f72870b6ae6dce58de4bec1b489f4bd65d80a8e326e1db8a"},
-    {file = "vowpal_wabbit_next-0.6.0-cp311-cp311-macosx_10_13_universal2.whl", hash = "sha256:3a8482d5c0b9357fdb36b62d659e6b74e93aeab165b910292572a98e91d7a014"},
-    {file = "vowpal_wabbit_next-0.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e4349099b938102f51fb6fedf035bc1deacb2971cd2a48641ca7d45186efda0"},
-    {file = "vowpal_wabbit_next-0.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:c8f58cdc49f270b1bed6f0fdd7520c8ba1b328de5cd8a2760c0ec70a630de92e"},
-    {file = "vowpal_wabbit_next-0.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8b7052ce7212fd1cae8ffd966e240c814f3c1df08fd612437d48f0f23e7694c"},
-    {file = "vowpal_wabbit_next-0.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d24d9c380d0e9b41151337c7f9e2a33ec5bfd738fdee9f65c1a40e486234aca3"},
-    {file = "vowpal_wabbit_next-0.6.0-cp38-cp38-macosx_10_13_universal2.whl", hash = "sha256:0d77a8c55249ec9a7f404939ecc6948db0527e522e8a7ae149ec7cd29b3ade04"},
-    {file = "vowpal_wabbit_next-0.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa2f52f1267fbc26c7757335f9c76a0f00b112971e04c85b8a9bc9e82300597"},
-    {file = "vowpal_wabbit_next-0.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:5d04f91200ecae73196d9f5601853d63afce8c1c8a0d310a608e8ddfa3b190cb"},
-    {file = "vowpal_wabbit_next-0.6.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:2df4a652729c0db34afd8fb4fc49b0090d6f061e2d49899e5f092fd4c3d23253"},
-    {file = "vowpal_wabbit_next-0.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c289a260ab759f04903b441701cff66ea74d6c061d966caaba0c65ac12d05528"},
-    {file = "vowpal_wabbit_next-0.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:8d022cab07274f227df159a81bccf034def7dd54ad70392ee98743ffa4953072"},
-]
-
-[package.dependencies]
-numpy = "*"
-
 [[package]]
 name = "watchdog"
 version = "3.0.0"
@@ -10509,4 +10482,4 @@ text-helpers = ["chardet"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8.1,<4.0"
-content-hash = "3707ffe51bf8aca6cca6512f9d3c358facbbc36a17a00ee9c7d513c557eddddb"
+content-hash = "b42a2fe9986973ebfe9804276429dff258c1124ef2df5985fa56ecd0cde7e7e1"

libs/langchain/pyproject.toml

@@ -129,7 +129,6 @@ markdownify = {version = "^0.11.6", optional = true}
 assemblyai = {version = "^0.17.0", optional = true}
 dashvector = {version = "^1.0.1", optional = true}
 sqlite-vss = {version = "^0.1.2", optional = true}
-vowpal-wabbit-next = {version = "0.6.0", optional = true}
 
 
 [tool.poetry.group.test.dependencies]
@@ -343,11 +342,9 @@ extended_testing = [
  "xmltodict",
  "faiss-cpu",
  "openapi-schema-pydantic",
- "sentence-transformers",
  "markdownify",
  "dashvector",
  "sqlite-vss",
- "vowpal-wabbit-next",
 ]
 
 [tool.ruff]