partners: AI21 Labs Jamba Support (#20815)

Description: Added support for AI21 new model - Jamba
Twitter handle: https://github.com/AI21Labs

---------

Co-authored-by: Asaf Gardin <asafg@ai21.com>
Co-authored-by: Erick Friis <erick@langchain.dev>

libs/partners/ai21/langchain_ai21/chat/chat_adapter.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Union, cast
+
+from ai21.models import ChatMessage as J2ChatMessage
+from ai21.models import RoleType
+from ai21.models.chat import ChatMessage
+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
+
+_ChatMessageTypes = Union[ChatMessage, J2ChatMessage]
+_SYSTEM_ERR_MESSAGE = "System message must be at beginning of message list."
+_ROLE_TYPE = Union[str, RoleType]
+
+
+class ChatAdapter(ABC):
+    """
+    Provides a common interface for the different Chat models available in AI21.
+    It converts LangChain messages to AI21 messages.
+    Calls the appropriate AI21 model API with the converted messages.
+    """
+
+    @abstractmethod
+    def convert_messages(
+        self,
+        messages: List[BaseMessage],
+    ) -> Dict[str, Any]:
+        pass
+
+    def _convert_message_to_ai21_message(
+        self,
+        message: BaseMessage,
+    ) -> _ChatMessageTypes:
+        content = cast(str, message.content)
+        role = self._parse_role(message)
+
+        return self._chat_message(role=role, content=content)
+
+    def _parse_role(self, message: BaseMessage) -> _ROLE_TYPE:
+        role = None
+
+        if isinstance(message, HumanMessage):
+            return RoleType.USER
+
+        if isinstance(message, AIMessage):
+            return RoleType.ASSISTANT
+
+        if isinstance(self, J2ChatAdapter):
+            if not role:
+                raise ValueError(
+                    f"Could not resolve role type from message {message}. "
+                    f"Only support {HumanMessage.__name__} and {AIMessage.__name__}."
+                )
+
+        # if it gets here, we rely on the server to handle the role type
+        return message.type
+
+    @abstractmethod
+    def _chat_message(
+        self,
+        role: _ROLE_TYPE,
+        content: str,
+    ) -> _ChatMessageTypes:
+        pass
+
+    @abstractmethod
+    def call(self, client: Any, **params: Any) -> List[BaseMessage]:
+        pass
+
+    def _get_system_message_from_message(self, message: BaseMessage) -> str:
+        if not isinstance(message.content, str):
+            raise ValueError(
+                f"System Message must be of type str. Got {type(message.content)}"
+            )
+
+        return message.content
+
+
+class J2ChatAdapter(ChatAdapter):
+    def convert_messages(self, messages: List[BaseMessage]) -> Dict[str, Any]:
+        system_message = ""
+        converted_messages = []  # type: ignore
+
+        for i, message in enumerate(messages):
+            if message.type == "system":
+                if i != 0:
+                    raise ValueError(_SYSTEM_ERR_MESSAGE)
+                else:
+                    system_message = self._get_system_message_from_message(message)
+            else:
+                converted_message = self._convert_message_to_ai21_message(message)
+                converted_messages.append(converted_message)
+
+        return {"system": system_message, "messages": converted_messages}
+
+    def _chat_message(
+        self,
+        role: _ROLE_TYPE,
+        content: str,
+    ) -> J2ChatMessage:
+        return J2ChatMessage(role=RoleType(role), text=content)
+
+    def call(self, client: Any, **params: Any) -> List[BaseMessage]:
+        response = client.chat.create(**params)
+
+        return [AIMessage(output.text) for output in response.outputs]
+
+
+class JambaChatCompletionsAdapter(ChatAdapter):
+    def convert_messages(self, messages: List[BaseMessage]) -> Dict[str, Any]:
+        return {
+            "messages": [
+                self._convert_message_to_ai21_message(message) for message in messages
+            ],
+        }
+
+    def _chat_message(
+        self,
+        role: _ROLE_TYPE,
+        content: str,
+    ) -> ChatMessage:
+        return ChatMessage(
+            role=role.value if isinstance(role, RoleType) else role,
+            content=content,
+        )
+
+    def call(self, client: Any, **params: Any) -> List[BaseMessage]:
+        response = client.chat.completions.create(**params)
+
+        return [AIMessage(choice.message.content) for choice in response.choices]

libs/partners/ai21/langchain_ai21/chat/chat_factory.py

@@ -0,0 +1,15 @@
+from langchain_ai21.chat.chat_adapter import (
+    ChatAdapter,
+    J2ChatAdapter,
+    JambaChatCompletionsAdapter,
+)
+
+
+def create_chat_adapter(model: str) -> ChatAdapter:
+    if "j2" in model:
+        return J2ChatAdapter()
+
+    if "jamba" in model:
+        return JambaChatCompletionsAdapter()
+
+    raise ValueError(f"Model {model} not supported.")

libs/partners/ai21/langchain_ai21/chat_models.py

@@ -1,79 +1,21 @@
 import asyncio
 from functools import partial
-from typing import Any, List, Mapping, Optional, Tuple, cast
+from typing import Any, Dict, List, Mapping, Optional
 
-from ai21.models import ChatMessage, RoleType
 from langchain_core.callbacks import (
     AsyncCallbackManagerForLLMRun,
     CallbackManagerForLLMRun,
 )
 from langchain_core.language_models.chat_models import BaseChatModel
 from langchain_core.messages import (
-    AIMessage,
     BaseMessage,
-    HumanMessage,
-    SystemMessage,
 )
 from langchain_core.outputs import ChatGeneration, ChatResult
+from langchain_core.pydantic_v1 import root_validator
 
 from langchain_ai21.ai21_base import AI21Base
-
-
-def _get_system_message_from_message(message: BaseMessage) -> str:
-    if not isinstance(message.content, str):
-        raise ValueError(
-            f"System Message must be of type str. Got {type(message.content)}"
-        )
-
-    return message.content
-
-
-def _convert_messages_to_ai21_messages(
-    messages: List[BaseMessage],
-) -> Tuple[Optional[str], List[ChatMessage]]:
-    system_message = None
-    converted_messages: List[ChatMessage] = []
-
-    for i, message in enumerate(messages):
-        if message.type == "system":
-            if i != 0:
-                raise ValueError("System message must be at beginning of message list.")
-            else:
-                system_message = _get_system_message_from_message(message)
-        else:
-            converted_message = _convert_message_to_ai21_message(message)
-            converted_messages.append(converted_message)
-
-    return system_message, converted_messages
-
-
-def _convert_message_to_ai21_message(
-    message: BaseMessage,
-) -> ChatMessage:
-    content = cast(str, message.content)
-
-    role = None
-
-    if isinstance(message, HumanMessage):
-        role = RoleType.USER
-    elif isinstance(message, AIMessage):
-        role = RoleType.ASSISTANT
-
-    if not role:
-        raise ValueError(
-            f"Could not resolve role type from message {message}. "
-            f"Only support {HumanMessage.__name__} and {AIMessage.__name__}."
-        )
-
-    return ChatMessage(role=role, text=content)
-
-
-def _pop_system_messages(messages: List[BaseMessage]) -> List[SystemMessage]:
-    system_message_indexes = [
-        i for i, message in enumerate(messages) if isinstance(message, SystemMessage)
-    ]
-
-    return [cast(SystemMessage, messages.pop(i)) for i in system_message_indexes]
+from langchain_ai21.chat.chat_adapter import ChatAdapter
+from langchain_ai21.chat.chat_factory import create_chat_adapter
 
 
 class ChatAI21(BaseChatModel, AI21Base):
@@ -119,6 +61,20 @@ class ChatAI21(BaseChatModel, AI21Base):
     """A penalty applied to tokens based on their frequency 
     in the generated responses."""
 
+    n: int = 1
+    """Number of chat completions to generate for each prompt."""
+
+    _chat_adapter: ChatAdapter
+
+    @root_validator()
+    def validate_environment(cls, values: Dict) -> Dict:
+        values = super().validate_environment(values)
+        model = values.get("model")
+
+        values["_chat_adapter"] = create_chat_adapter(model)  # type: ignore
+
+        return values
+
     class Config:
         """Configuration for this pydantic object."""
 
@@ -139,6 +95,7 @@ class ChatAI21(BaseChatModel, AI21Base):
             "temperature": self.temperature,
             "top_p": self.top_p,
             "top_k_return": self.top_k_return,
+            "n": self.n,
         }
 
         if self.count_penalty is not None:
@@ -159,7 +116,7 @@ class ChatAI21(BaseChatModel, AI21Base):
         **kwargs: Any,
     ) -> Mapping[str, Any]:
         params = {}
-        system, ai21_messages = _convert_messages_to_ai21_messages(messages)
+        converted_messages = self._chat_adapter.convert_messages(messages)
 
         if stop is not None:
             if "stop" in kwargs:
@@ -167,8 +124,7 @@ class ChatAI21(BaseChatModel, AI21Base):
             params["stop_sequences"] = stop
 
         return {
-            "system": system or "",
-            "messages": ai21_messages,
+            **converted_messages,
             **self._default_params,
             **params,
             **kwargs,
@@ -182,12 +138,10 @@ class ChatAI21(BaseChatModel, AI21Base):
         **kwargs: Any,
     ) -> ChatResult:
         params = self._build_params_for_request(messages=messages, stop=stop, **kwargs)
+        messages = self._chat_adapter.call(self.client, **params)
+        generations = [ChatGeneration(message=message) for message in messages]
 
-        response = self.client.chat.create(**params)
-
-        outputs = response.outputs
-        message = AIMessage(content=outputs[0].text)
-        return ChatResult(generations=[ChatGeneration(message=message)])
+        return ChatResult(generations=generations)
 
     async def _agenerate(
         self,
@@ -199,3 +153,11 @@ class ChatAI21(BaseChatModel, AI21Base):
         return await asyncio.get_running_loop().run_in_executor(
             None, partial(self._generate, **kwargs), messages, stop, run_manager
         )
+
+    def _get_system_message_from_message(self, message: BaseMessage) -> str:
+        if not isinstance(message.content, str):
+            raise ValueError(
+                f"System Message must be of type str. Got {type(message.content)}"
+            )
+
+        return message.content

libs/partners/ai21/poetry.lock

@@ -2,17 +2,17 @@
 
 [[package]]
 name = "ai21"
-version = "2.1.3"
+version = "2.2.1"
 description = ""
 optional = false
 python-versions = "<4.0,>=3.8"
 files = [
-    {file = "ai21-2.1.3-py3-none-any.whl", hash = "sha256:384f3b5769edfb7124e9288bc83daeeeee35f6f831be2f77425c061774d5ade3"},
-    {file = "ai21-2.1.3.tar.gz", hash = "sha256:80014f54818453e87ced5c3d180e22a6cfb25911231ec834cae55f8769106afe"},
+    {file = "ai21-2.2.1-py3-none-any.whl", hash = "sha256:c2727656b07a1284fc8f1f2fb1b065059f6fa5fcab8af2f45b0767086a044771"},
+    {file = "ai21-2.2.1.tar.gz", hash = "sha256:69e659071e9bbadae7349465bc388312bcff30e22fb7d7a9de4319e2925acfff"},
 ]
 
 [package.dependencies]
-ai21-tokenizer = ">=0.3.9,<0.4.0"
+ai21-tokenizer = ">=0.9.0,<0.10.0"
 dataclasses-json = ">=0.6.3,<0.7.0"
 requests = ">=2.31.0,<3.0.0"
 typing-extensions = ">=4.9.0,<5.0.0"
@@ -22,17 +22,18 @@ aws = ["boto3 (>=1.28.82,<2.0.0)"]
 
 [[package]]
 name = "ai21-tokenizer"
-version = "0.3.11"
+version = "0.9.0"
 description = ""
 optional = false
-python-versions = ">=3.7,<4.0"
+python-versions = "<4.0,>=3.7"
 files = [
-    {file = "ai21_tokenizer-0.3.11-py3-none-any.whl", hash = "sha256:80d332c51cab3fa88f0fea7493240a6a5bc38fd24a3d0806d28731d8fc97691f"},
-    {file = "ai21_tokenizer-0.3.11.tar.gz", hash = "sha256:ec11ce4e46d24f71f1c2756ad0de34e0adfd51b5bcd81b544aea13d6935ec905"},
+    {file = "ai21_tokenizer-0.9.0-py3-none-any.whl", hash = "sha256:3e4927f0ef98923f53710405fb476b29102bc7c677c65243043c5671cb642d9e"},
+    {file = "ai21_tokenizer-0.9.0.tar.gz", hash = "sha256:724d368eb74564950edfcd6590da7c3b702c4a8f59f60d36dbec5046e6856579"},
 ]
 
 [package.dependencies]
-sentencepiece = ">=0.1.96,<0.2.0"
+sentencepiece = ">=0.2.0,<0.3.0"
+tokenizers = ">=0.15.2,<0.16.0"
 
 [[package]]
 name = "annotated-types"
@@ -215,6 +216,22 @@ files = [
 [package.extras]
 test = ["pytest (>=6)"]
 
+[[package]]
+name = "filelock"
+version = "3.13.4"
+description = "A platform independent file lock."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "filelock-3.13.4-py3-none-any.whl", hash = "sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f"},
+    {file = "filelock-3.13.4.tar.gz", hash = "sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4"},
+]
+
+[package.extras]
+docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"]
+testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"]
+typing = ["typing-extensions (>=4.8)"]
+
 [[package]]
 name = "freezegun"
 version = "1.4.0"
@@ -229,6 +246,75 @@ files = [
 [package.dependencies]
 python-dateutil = ">=2.7"
 
+[[package]]
+name = "fsspec"
+version = "2024.3.1"
+description = "File-system specification"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "fsspec-2024.3.1-py3-none-any.whl", hash = "sha256:918d18d41bf73f0e2b261824baeb1b124bcf771767e3a26425cd7dec3332f512"},
+    {file = "fsspec-2024.3.1.tar.gz", hash = "sha256:f39780e282d7d117ffb42bb96992f8a90795e4d0fb0f661a70ca39fe9c43ded9"},
+]
+
+[package.extras]
+abfs = ["adlfs"]
+adl = ["adlfs"]
+arrow = ["pyarrow (>=1)"]
+dask = ["dask", "distributed"]
+devel = ["pytest", "pytest-cov"]
+dropbox = ["dropbox", "dropboxdrivefs", "requests"]
+full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"]
+fuse = ["fusepy"]
+gcs = ["gcsfs"]
+git = ["pygit2"]
+github = ["requests"]
+gs = ["gcsfs"]
+gui = ["panel"]
+hdfs = ["pyarrow (>=1)"]
+http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"]
+libarchive = ["libarchive-c"]
+oci = ["ocifs"]
+s3 = ["s3fs"]
+sftp = ["paramiko"]
+smb = ["smbprotocol"]
+ssh = ["paramiko"]
+tqdm = ["tqdm"]
+
+[[package]]
+name = "huggingface-hub"
+version = "0.22.2"
+description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"
+optional = false
+python-versions = ">=3.8.0"
+files = [
+    {file = "huggingface_hub-0.22.2-py3-none-any.whl", hash = "sha256:3429e25f38ccb834d310804a3b711e7e4953db5a9e420cc147a5e194ca90fd17"},
+    {file = "huggingface_hub-0.22.2.tar.gz", hash = "sha256:32e9a9a6843c92f253ff9ca16b9985def4d80a93fb357af5353f770ef74a81be"},
+]
+
+[package.dependencies]
+filelock = "*"
+fsspec = ">=2023.5.0"
+packaging = ">=20.9"
+pyyaml = ">=5.1"
+requests = "*"
+tqdm = ">=4.42.1"
+typing-extensions = ">=3.7.4.3"
+
+[package.extras]
+all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "minijinja (>=1.0)", "mypy (==1.5.1)", "numpy", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.3.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"]
+cli = ["InquirerPy (==0.3.4)"]
+dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "minijinja (>=1.0)", "mypy (==1.5.1)", "numpy", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.3.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"]
+fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"]
+hf-transfer = ["hf-transfer (>=0.1.4)"]
+inference = ["aiohttp", "minijinja (>=1.0)"]
+quality = ["mypy (==1.5.1)", "ruff (>=0.3.0)"]
+tensorflow = ["graphviz", "pydot", "tensorflow"]
+tensorflow-testing = ["keras (<3.0)", "tensorflow"]
+testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "minijinja (>=1.0)", "numpy", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"]
+torch = ["safetensors", "torch"]
+typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"]
+
 [[package]]
 name = "idna"
 version = "3.6"
@@ -278,7 +364,7 @@ files = [
 
 [[package]]
 name = "langchain-core"
-version = "0.1.42"
+version = "0.1.45"
 description = "Building applications with LLMs through composability"
 optional = false
 python-versions = ">=3.8.1,<4.0"
@@ -737,7 +823,6 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
     {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
     {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
     {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -821,56 +906,64 @@ files = [
 
 [[package]]
 name = "sentencepiece"
-version = "0.1.99"
+version = "0.2.0"
 description = "SentencePiece python wrapper"
 optional = false
 python-versions = "*"
 files = [
-    {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0eb528e70571b7c02723e5804322469b82fe7ea418c96051d0286c0fa028db73"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:77d7fafb2c4e4659cbdf303929503f37a26eabc4ff31d3a79bf1c5a1b338caa7"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be9cf5b9e404c245aeb3d3723c737ba7a8f5d4ba262ef233a431fa6c45f732a0"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baed1a26464998f9710d20e52607c29ffd4293e7c71c6a1f83f51ad0911ec12c"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9832f08bb372d4c8b567612f8eab9e36e268dff645f1c28f9f8e851be705f6d1"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:019e7535108e309dae2b253a75834fc3128240aa87c00eb80732078cdc182588"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-win32.whl", hash = "sha256:fa16a830416bb823fa2a52cbdd474d1f7f3bba527fd2304fb4b140dad31bb9bc"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-win_amd64.whl", hash = "sha256:14b0eccb7b641d4591c3e12ae44cab537d68352e4d3b6424944f0c447d2348d5"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6d3c56f24183a1e8bd61043ff2c58dfecdc68a5dd8955dc13bab83afd5f76b81"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed6ea1819fd612c989999e44a51bf556d0ef6abfb553080b9be3d347e18bcfb7"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2a0260cd1fb7bd8b4d4f39dc2444a8d5fd4e0a0c4d5c899810ef1abf99b2d45"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a1abff4d1ff81c77cac3cc6fefa34fa4b8b371e5ee51cb7e8d1ebc996d05983"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:004e6a621d4bc88978eecb6ea7959264239a17b70f2cbc348033d8195c9808ec"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db361e03342c41680afae5807590bc88aa0e17cfd1a42696a160e4005fcda03b"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-win32.whl", hash = "sha256:2d95e19168875b70df62916eb55428a0cbcb834ac51d5a7e664eda74def9e1e0"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-win_amd64.whl", hash = "sha256:f90d73a6f81248a909f55d8e6ef56fec32d559e1e9af045f0b0322637cb8e5c7"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:62e24c81e74bd87a6e0d63c51beb6527e4c0add67e1a17bac18bcd2076afcfeb"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57efcc2d51caff20d9573567d9fd3f854d9efe613ed58a439c78c9f93101384a"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a904c46197993bd1e95b93a6e373dca2f170379d64441041e2e628ad4afb16f"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d89adf59854741c0d465f0e1525b388c0d174f611cc04af54153c5c4f36088c4"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-win32.whl", hash = "sha256:47c378146928690d1bc106fdf0da768cebd03b65dd8405aa3dd88f9c81e35dba"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-win_amd64.whl", hash = "sha256:9ba142e7a90dd6d823c44f9870abdad45e6c63958eb60fe44cca6828d3b69da2"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b7b1a9ae4d7c6f1f867e63370cca25cc17b6f4886729595b885ee07a58d3cec3"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0f644c9d4d35c096a538507b2163e6191512460035bf51358794a78515b74f7"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8843d23a0f686d85e569bd6dcd0dd0e0cbc03731e63497ca6d5bacd18df8b85"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e6f690a1caebb4867a2e367afa1918ad35be257ecdb3455d2bbd787936f155"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-win32.whl", hash = "sha256:8a321866c2f85da7beac74a824b4ad6ddc2a4c9bccd9382529506d48f744a12c"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-win_amd64.whl", hash = "sha256:c42f753bcfb7661c122a15b20be7f684b61fc8592c89c870adf52382ea72262d"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:85b476406da69c70586f0bb682fcca4c9b40e5059814f2db92303ea4585c650c"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cfbcfe13c69d3f87b7fcd5da168df7290a6d006329be71f90ba4f56bc77f8561"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:445b0ec381af1cd4eef95243e7180c63d9c384443c16c4c47a28196bd1cda937"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6890ea0f2b4703f62d0bf27932e35808b1f679bdb05c7eeb3812b935ba02001"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb71af492b0eefbf9f2501bec97bcd043b6812ab000d119eaf4bd33f9e283d03"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27b866b5bd3ddd54166bbcbf5c8d7dd2e0b397fac8537991c7f544220b1f67bc"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-win32.whl", hash = "sha256:b133e8a499eac49c581c3c76e9bdd08c338cc1939e441fee6f92c0ccb5f1f8be"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-win_amd64.whl", hash = "sha256:0eaf3591dd0690a87f44f4df129cf8d05d8a4029b5b6709b489b8e27f9a9bcff"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38efeda9bbfb55052d482a009c6a37e52f42ebffcea9d3a98a61de7aee356a28"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6c030b081dc1e1bcc9fadc314b19b740715d3d566ad73a482da20d7d46fd444c"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84dbe53e02e4f8a2e45d2ac3e430d5c83182142658e25edd76539b7648928727"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b0f55d0a0ee1719b4b04221fe0c9f0c3461dc3dabd77a035fa2f4788eb3ef9a"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e800f206cd235dc27dc749299e05853a4e4332e8d3dfd81bf13d0e5b9007d9"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ae1c40cda8f9d5b0423cfa98542735c0235e7597d79caf318855cdf971b2280"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-win32.whl", hash = "sha256:c84ce33af12ca222d14a1cdd37bd76a69401e32bc68fe61c67ef6b59402f4ab8"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-win_amd64.whl", hash = "sha256:350e5c74d739973f1c9643edb80f7cc904dc948578bcb1d43c6f2b173e5d18dd"},
-    {file = "sentencepiece-0.1.99.tar.gz", hash = "sha256:189c48f5cb2949288f97ccdb97f0473098d9c3dcf5a3d99d4eabe719ec27297f"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:188779e1298a1c8b8253c7d3ad729cb0a9891e5cef5e5d07ce4592c54869e227"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bed9cf85b296fa2b76fc2547b9cbb691a523864cebaee86304c43a7b4cb1b452"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d7b67e724bead13f18db6e1d10b6bbdc454af574d70efbb36f27d90387be1ca3"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fde4b08cfe237be4484c6c7c2e2c75fb862cfeab6bd5449ce4caeafd97b767a"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c378492056202d1c48a4979650981635fd97875a00eabb1f00c6a236b013b5e"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1380ce6540a368de2ef6d7e6ba14ba8f3258df650d39ba7d833b79ee68a52040"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-win32.whl", hash = "sha256:a1151d6a6dd4b43e552394aed0edfe9292820272f0194bd56c7c1660a0c06c3d"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:d490142b0521ef22bc1085f061d922a2a6666175bb6b42e588ff95c0db6819b2"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:17982700c4f6dbb55fa3594f3d7e5dd1c8659a274af3738e33c987d2a27c9d5c"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7c867012c0e8bcd5bdad0f791609101cb5c66acb303ab3270218d6debc68a65e"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7fd6071249c74f779c5b27183295b9202f8dedb68034e716784364443879eaa6"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f90c55a65013cbb8f4d7aab0599bf925cde4adc67ae43a0d323677b5a1c6cb"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b293734059ef656dcd65be62ff771507bea8fed0a711b6733976e1ed3add4553"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e58b47f933aca74c6a60a79dcb21d5b9e47416256c795c2d58d55cec27f9551d"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-win32.whl", hash = "sha256:c581258cf346b327c62c4f1cebd32691826306f6a41d8c4bec43b010dee08e75"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:0993dbc665f4113017892f1b87c3904a44d0640eda510abcacdfb07f74286d36"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ea5f536e32ea8ec96086ee00d7a4a131ce583a1b18d130711707c10e69601cb2"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d0cb51f53b6aae3c36bafe41e86167c71af8370a039f542c43b0cce5ef24a68c"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3212121805afc58d8b00ab4e7dd1f8f76c203ddb9dc94aa4079618a31cf5da0f"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a3149e3066c2a75e0d68a43eb632d7ae728c7925b517f4c05c40f6f7280ce08"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:632f3594d3e7ac8b367bca204cb3fd05a01d5b21455acd097ea4c0e30e2f63d7"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f295105c6bdbb05bd5e1b0cafbd78ff95036f5d3641e7949455a3f4e5e7c3109"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-win32.whl", hash = "sha256:fb89f811e5efd18bab141afc3fea3de141c3f69f3fe9e898f710ae7fe3aab251"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7a673a72aab81fef5ebe755c6e0cc60087d1f3a4700835d40537183c1703a45f"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4547683f330289ec4f093027bfeb87f9ef023b2eb6f879fdc4a8187c7e0ffb90"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cd6175f7eaec7142d2bf6f6597ce7db4c9ac89acf93fcdb17410c3a8b781eeb"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:859ba1acde782609a0910a26a60e16c191a82bf39b5621107552c0cd79fad00f"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcbbef6cc277f8f18f36959e305f10b1c620442d75addc79c21d7073ae581b50"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-win32.whl", hash = "sha256:536b934e244829e3fe6c4f198652cd82da48adb9aa145c9f00889542726dee3d"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:0a91aaa3c769b52440df56fafda683b3aa48e3f2169cf7ee5b8c8454a7f3ae9b"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:787e480ca4c1d08c9985a7eb1eae4345c107729c99e9b5a9a00f2575fc7d4b4b"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4d158189eb2ecffea3a51edf6d25e110b3678ec47f1a40f2d541eafbd8f6250"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1e5ca43013e8935f25457a4fca47e315780172c3e821b4b13a890668911c792"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7140d9e5a74a0908493bb4a13f1f16a401297bd755ada4c707e842fbf6f0f5bf"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-win32.whl", hash = "sha256:6cf333625234f247ab357b0bd9836638405ea9082e1543d5b8408f014979dcbf"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ff88712338b01031910e8e61e7239aff3ce8869ee31a47df63cb38aadd591bea"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20813a68d4c221b1849c62c30e1281ea81687894d894b8d4a0f4677d9311e0f5"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:926ef920ae2e8182db31d3f5d081ada57804e3e1d3a8c4ef8b117f9d9fb5a945"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:89f65f69636b7e9c015b79dff9c9985a9bc7d19ded6f79ef9f1ec920fdd73ecf"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f67eae0dbe6f2d7d6ba50a354623d787c99965f068b81e145d53240198021b0"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:98501e075f35dd1a1d5a20f65be26839fcb1938752ec61539af008a5aa6f510b"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3d1d2cc4882e8d6a1adf9d5927d7716f80617fc693385661caff21888972269"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-win32.whl", hash = "sha256:b99a308a2e5e569031ab164b74e6fab0b6f37dfb493c32f7816225f4d411a6dd"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:cdb701eec783d3ec86b7cd4c763adad8eaf6b46db37ee1c36e5e6c44b3fe1b5f"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1e0f9c4d0a6b0af59b613175f019916e28ade076e21242fd5be24340d8a2f64a"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:298f21cc1366eb60311aedba3169d30f885c363ddbf44214b0a587d2908141ad"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3f1ec95aa1e5dab11f37ac7eff190493fd87770f7a8b81ebc9dd768d1a3c8704"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b06b70af54daa4b4904cbb90b4eb6d35c9f3252fdc86c9c32d5afd4d30118d8"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22e37bac44dd6603388cb598c64ff7a76e41ca774646f21c23aadfbf5a2228ab"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0461324897735512a32d222e3d886e24ad6a499761952b6bda2a9ee6e4313ea5"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-win32.whl", hash = "sha256:38aed822fb76435fa1f12185f10465a94ab9e51d5e8a9159e9a540ce926f0ffd"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:d8cf876516548b5a1d6ac4745d8b554f5c07891d55da557925e5c13ff0b4e6ad"},
+    {file = "sentencepiece-0.2.0.tar.gz", hash = "sha256:a52c19171daaf2e697dc6cbe67684e0fa341b1248966f6aebb541de654d15843"},
 ]
 
 [[package]]
@@ -912,6 +1005,133 @@ files = [
 [package.extras]
 doc = ["reno", "sphinx", "tornado (>=4.5)"]
 
+[[package]]
+name = "tokenizers"
+version = "0.15.2"
+description = ""
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "tokenizers-0.15.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:52f6130c9cbf70544287575a985bf44ae1bda2da7e8c24e97716080593638012"},
+    {file = "tokenizers-0.15.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:054c1cc9c6d68f7ffa4e810b3d5131e0ba511b6e4be34157aa08ee54c2f8d9ee"},
+    {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a9b9b070fdad06e347563b88c278995735292ded1132f8657084989a4c84a6d5"},
+    {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea621a7eef4b70e1f7a4e84dd989ae3f0eeb50fc8690254eacc08acb623e82f1"},
+    {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf7fd9a5141634fa3aa8d6b7be362e6ae1b4cda60da81388fa533e0b552c98fd"},
+    {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44f2a832cd0825295f7179eaf173381dc45230f9227ec4b44378322d900447c9"},
+    {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b9ec69247a23747669ec4b0ca10f8e3dfb3545d550258129bd62291aabe8605"},
+    {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40b6a4c78da863ff26dbd5ad9a8ecc33d8a8d97b535172601cf00aee9d7ce9ce"},
+    {file = "tokenizers-0.15.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5ab2a4d21dcf76af60e05af8063138849eb1d6553a0d059f6534357bce8ba364"},
+    {file = "tokenizers-0.15.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a47acfac7e511f6bbfcf2d3fb8c26979c780a91e06fb5b9a43831b2c0153d024"},
+    {file = "tokenizers-0.15.2-cp310-none-win32.whl", hash = "sha256:064ff87bb6acdbd693666de9a4b692add41308a2c0ec0770d6385737117215f2"},
+    {file = "tokenizers-0.15.2-cp310-none-win_amd64.whl", hash = "sha256:3b919afe4df7eb6ac7cafd2bd14fb507d3f408db7a68c43117f579c984a73843"},
+    {file = "tokenizers-0.15.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:89cd1cb93e4b12ff39bb2d626ad77e35209de9309a71e4d3d4672667b4b256e7"},
+    {file = "tokenizers-0.15.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cfed5c64e5be23d7ee0f0e98081a25c2a46b0b77ce99a4f0605b1ec43dd481fa"},
+    {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a907d76dcfda37023ba203ab4ceeb21bc5683436ebefbd895a0841fd52f6f6f2"},
+    {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20ea60479de6fc7b8ae756b4b097572372d7e4032e2521c1bbf3d90c90a99ff0"},
+    {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:48e2b9335be2bc0171df9281385c2ed06a15f5cf121c44094338306ab7b33f2c"},
+    {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:112a1dd436d2cc06e6ffdc0b06d55ac019a35a63afd26475205cb4b1bf0bfbff"},
+    {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4620cca5c2817177ee8706f860364cc3a8845bc1e291aaf661fb899e5d1c45b0"},
+    {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccd73a82751c523b3fc31ff8194702e4af4db21dc20e55b30ecc2079c5d43cb7"},
+    {file = "tokenizers-0.15.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:107089f135b4ae7817affe6264f8c7a5c5b4fd9a90f9439ed495f54fcea56fb4"},
+    {file = "tokenizers-0.15.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0ff110ecc57b7aa4a594396525a3451ad70988e517237fe91c540997c4e50e29"},
+    {file = "tokenizers-0.15.2-cp311-none-win32.whl", hash = "sha256:6d76f00f5c32da36c61f41c58346a4fa7f0a61be02f4301fd30ad59834977cc3"},
+    {file = "tokenizers-0.15.2-cp311-none-win_amd64.whl", hash = "sha256:cc90102ed17271cf0a1262babe5939e0134b3890345d11a19c3145184b706055"},
+    {file = "tokenizers-0.15.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f86593c18d2e6248e72fb91c77d413a815153b8ea4e31f7cd443bdf28e467670"},
+    {file = "tokenizers-0.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0774bccc6608eca23eb9d620196687c8b2360624619623cf4ba9dc9bd53e8b51"},
+    {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d0222c5b7c9b26c0b4822a82f6a7011de0a9d3060e1da176f66274b70f846b98"},
+    {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3835738be1de66624fff2f4f6f6684775da4e9c00bde053be7564cbf3545cc66"},
+    {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0143e7d9dcd811855c1ce1ab9bf5d96d29bf5e528fd6c7824d0465741e8c10fd"},
+    {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db35825f6d54215f6b6009a7ff3eedee0848c99a6271c870d2826fbbedf31a38"},
+    {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f5e64b0389a2be47091d8cc53c87859783b837ea1a06edd9d8e04004df55a5c"},
+    {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e0480c452217edd35eca56fafe2029fb4d368b7c0475f8dfa3c5c9c400a7456"},
+    {file = "tokenizers-0.15.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a33ab881c8fe70474980577e033d0bc9a27b7ab8272896e500708b212995d834"},
+    {file = "tokenizers-0.15.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a308a607ca9de2c64c1b9ba79ec9a403969715a1b8ba5f998a676826f1a7039d"},
+    {file = "tokenizers-0.15.2-cp312-none-win32.whl", hash = "sha256:b8fcfa81bcb9447df582c5bc96a031e6df4da2a774b8080d4f02c0c16b42be0b"},
+    {file = "tokenizers-0.15.2-cp312-none-win_amd64.whl", hash = "sha256:38d7ab43c6825abfc0b661d95f39c7f8af2449364f01d331f3b51c94dcff7221"},
+    {file = "tokenizers-0.15.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:38bfb0204ff3246ca4d5e726e8cc8403bfc931090151e6eede54d0e0cf162ef0"},
+    {file = "tokenizers-0.15.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c861d35e8286a53e06e9e28d030b5a05bcbf5ac9d7229e561e53c352a85b1fc"},
+    {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:936bf3842db5b2048eaa53dade907b1160f318e7c90c74bfab86f1e47720bdd6"},
+    {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:620beacc3373277700d0e27718aa8b25f7b383eb8001fba94ee00aeea1459d89"},
+    {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2735ecbbf37e52db4ea970e539fd2d450d213517b77745114f92867f3fc246eb"},
+    {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:473c83c5e2359bb81b0b6fde870b41b2764fcdd36d997485e07e72cc3a62264a"},
+    {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968fa1fb3c27398b28a4eca1cbd1e19355c4d3a6007f7398d48826bbe3a0f728"},
+    {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:865c60ae6eaebdde7da66191ee9b7db52e542ed8ee9d2c653b6d190a9351b980"},
+    {file = "tokenizers-0.15.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7c0d8b52664ab2d4a8d6686eb5effc68b78608a9008f086a122a7b2996befbab"},
+    {file = "tokenizers-0.15.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f33dfbdec3784093a9aebb3680d1f91336c56d86cc70ddf88708251da1fe9064"},
+    {file = "tokenizers-0.15.2-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d44ba80988ff9424e33e0a49445072ac7029d8c0e1601ad25a0ca5f41ed0c1d6"},
+    {file = "tokenizers-0.15.2-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:dce74266919b892f82b1b86025a613956ea0ea62a4843d4c4237be2c5498ed3a"},
+    {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0ef06b9707baeb98b316577acb04f4852239d856b93e9ec3a299622f6084e4be"},
+    {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c73e2e74bbb07910da0d37c326869f34113137b23eadad3fc00856e6b3d9930c"},
+    {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4eeb12daf02a59e29f578a865f55d87cd103ce62bd8a3a5874f8fdeaa82e336b"},
+    {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ba9f6895af58487ca4f54e8a664a322f16c26bbb442effd01087eba391a719e"},
+    {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ccec77aa7150e38eec6878a493bf8c263ff1fa8a62404e16c6203c64c1f16a26"},
+    {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3f40604f5042ff210ba82743dda2b6aa3e55aa12df4e9f2378ee01a17e2855e"},
+    {file = "tokenizers-0.15.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5645938a42d78c4885086767c70923abad047163d809c16da75d6b290cb30bbe"},
+    {file = "tokenizers-0.15.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:05a77cbfebe28a61ab5c3891f9939cc24798b63fa236d84e5f29f3a85a200c00"},
+    {file = "tokenizers-0.15.2-cp37-none-win32.whl", hash = "sha256:361abdc068e8afe9c5b818769a48624687fb6aaed49636ee39bec4e95e1a215b"},
+    {file = "tokenizers-0.15.2-cp37-none-win_amd64.whl", hash = "sha256:7ef789f83eb0f9baeb4d09a86cd639c0a5518528f9992f38b28e819df397eb06"},
+    {file = "tokenizers-0.15.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4fe1f74a902bee74a3b25aff180fbfbf4f8b444ab37c4d496af7afd13a784ed2"},
+    {file = "tokenizers-0.15.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c4b89038a684f40a6b15d6b09f49650ac64d951ad0f2a3ea9169687bbf2a8ba"},
+    {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d05a1b06f986d41aed5f2de464c003004b2df8aaf66f2b7628254bcbfb72a438"},
+    {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:508711a108684111ec8af89d3a9e9e08755247eda27d0ba5e3c50e9da1600f6d"},
+    {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:daa348f02d15160cb35439098ac96e3a53bacf35885072611cd9e5be7d333daa"},
+    {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:494fdbe5932d3416de2a85fc2470b797e6f3226c12845cadf054dd906afd0442"},
+    {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2d60f5246f4da9373f75ff18d64c69cbf60c3bca597290cea01059c336d2470"},
+    {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93268e788825f52de4c7bdcb6ebc1fcd4a5442c02e730faa9b6b08f23ead0e24"},
+    {file = "tokenizers-0.15.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6fc7083ab404019fc9acafe78662c192673c1e696bd598d16dc005bd663a5cf9"},
+    {file = "tokenizers-0.15.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:41e39b41e5531d6b2122a77532dbea60e171ef87a3820b5a3888daa847df4153"},
+    {file = "tokenizers-0.15.2-cp38-none-win32.whl", hash = "sha256:06cd0487b1cbfabefb2cc52fbd6b1f8d4c37799bd6c6e1641281adaa6b2504a7"},
+    {file = "tokenizers-0.15.2-cp38-none-win_amd64.whl", hash = "sha256:5179c271aa5de9c71712e31cb5a79e436ecd0d7532a408fa42a8dbfa4bc23fd9"},
+    {file = "tokenizers-0.15.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:82f8652a74cc107052328b87ea8b34291c0f55b96d8fb261b3880216a9f9e48e"},
+    {file = "tokenizers-0.15.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:02458bee6f5f3139f1ebbb6d042b283af712c0981f5bc50edf771d6b762d5e4f"},
+    {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c9a09cd26cca2e1c349f91aa665309ddb48d71636370749414fbf67bc83c5343"},
+    {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:158be8ea8554e5ed69acc1ce3fbb23a06060bd4bbb09029431ad6b9a466a7121"},
+    {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ddba9a2b0c8c81633eca0bb2e1aa5b3a15362b1277f1ae64176d0f6eba78ab1"},
+    {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ef5dd1d39797044642dbe53eb2bc56435308432e9c7907728da74c69ee2adca"},
+    {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:454c203164e07a860dbeb3b1f4a733be52b0edbb4dd2e5bd75023ffa8b49403a"},
+    {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cf6b7f1d4dc59af960e6ffdc4faffe6460bbfa8dce27a58bf75755ffdb2526d"},
+    {file = "tokenizers-0.15.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2ef09bbc16519f6c25d0c7fc0c6a33a6f62923e263c9d7cca4e58b8c61572afb"},
+    {file = "tokenizers-0.15.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c9a2ebdd2ad4ec7a68e7615086e633857c85e2f18025bd05d2a4399e6c5f7169"},
+    {file = "tokenizers-0.15.2-cp39-none-win32.whl", hash = "sha256:918fbb0eab96fe08e72a8c2b5461e9cce95585d82a58688e7f01c2bd546c79d0"},
+    {file = "tokenizers-0.15.2-cp39-none-win_amd64.whl", hash = "sha256:524e60da0135e106b254bd71f0659be9f89d83f006ea9093ce4d1fab498c6d0d"},
+    {file = "tokenizers-0.15.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6a9b648a58281c4672212fab04e60648fde574877d0139cd4b4f93fe28ca8944"},
+    {file = "tokenizers-0.15.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7c7d18b733be6bbca8a55084027f7be428c947ddf871c500ee603e375013ffba"},
+    {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:13ca3611de8d9ddfbc4dc39ef54ab1d2d4aaa114ac8727dfdc6a6ec4be017378"},
+    {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:237d1bf3361cf2e6463e6c140628e6406766e8b27274f5fcc62c747ae3c6f094"},
+    {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67a0fe1e49e60c664915e9fb6b0cb19bac082ab1f309188230e4b2920230edb3"},
+    {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4e022fe65e99230b8fd89ebdfea138c24421f91c1a4f4781a8f5016fd5cdfb4d"},
+    {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d857be2df69763362ac699f8b251a8cd3fac9d21893de129bc788f8baaef2693"},
+    {file = "tokenizers-0.15.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:708bb3e4283177236309e698da5fcd0879ce8fd37457d7c266d16b550bcbbd18"},
+    {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c35e09e9899b72a76e762f9854e8750213f67567787d45f37ce06daf57ca78"},
+    {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1257f4394be0d3b00de8c9e840ca5601d0a4a8438361ce9c2b05c7d25f6057b"},
+    {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02272fe48280e0293a04245ca5d919b2c94a48b408b55e858feae9618138aeda"},
+    {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dc3ad9ebc76eabe8b1d7c04d38be884b8f9d60c0cdc09b0aa4e3bcf746de0388"},
+    {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:32e16bdeffa7c4f46bf2152172ca511808b952701d13e7c18833c0b73cb5c23f"},
+    {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fb16ba563d59003028b678d2361a27f7e4ae0ab29c7a80690efa20d829c81fdb"},
+    {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:2277c36d2d6cdb7876c274547921a42425b6810d38354327dd65a8009acf870c"},
+    {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1cf75d32e8d250781940d07f7eece253f2fe9ecdb1dc7ba6e3833fa17b82fcbc"},
+    {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b3b31884dc8e9b21508bb76da80ebf7308fdb947a17affce815665d5c4d028"},
+    {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10122d8d8e30afb43bb1fe21a3619f62c3e2574bff2699cf8af8b0b6c5dc4a3"},
+    {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d88b96ff0fe8e91f6ef01ba50b0d71db5017fa4e3b1d99681cec89a85faf7bf7"},
+    {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:37aaec5a52e959892870a7c47cef80c53797c0db9149d458460f4f31e2fb250e"},
+    {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e2ea752f2b0fe96eb6e2f3adbbf4d72aaa1272079b0dfa1145507bd6a5d537e6"},
+    {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4b19a808d8799fda23504a5cd31d2f58e6f52f140380082b352f877017d6342b"},
+    {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c86e5e068ac8b19204419ed8ca90f9d25db20578f5881e337d203b314f4104"},
+    {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de19c4dc503c612847edf833c82e9f73cd79926a384af9d801dcf93f110cea4e"},
+    {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea09acd2fe3324174063d61ad620dec3bcf042b495515f27f638270a7d466e8b"},
+    {file = "tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cf27fd43472e07b57cf420eee1e814549203d56de00b5af8659cb99885472f1f"},
+    {file = "tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7ca22bd897537a0080521445d91a58886c8c04084a6a19e6c78c586e0cfa92a5"},
+    {file = "tokenizers-0.15.2.tar.gz", hash = "sha256:e6e9c6e019dd5484be5beafc775ae6c925f4c69a3487040ed09b45e13df2cb91"},
+]
+
+[package.dependencies]
+huggingface_hub = ">=0.16.4,<1.0"
+
+[package.extras]
+dev = ["tokenizers[testing]"]
+docs = ["setuptools_rust", "sphinx", "sphinx_rtd_theme"]
+testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"]
+
 [[package]]
 name = "tomli"
 version = "2.0.1"
@@ -923,6 +1143,26 @@ files = [
     {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
 ]
 
+[[package]]
+name = "tqdm"
+version = "4.66.2"
+description = "Fast, Extensible Progress Meter"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "tqdm-4.66.2-py3-none-any.whl", hash = "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9"},
+    {file = "tqdm-4.66.2.tar.gz", hash = "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[package.extras]
+dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"]
+notebook = ["ipywidgets (>=6)"]
+slack = ["slack-sdk"]
+telegram = ["requests"]
+
 [[package]]
 name = "typing-extensions"
 version = "4.10.0"
@@ -1010,4 +1250,4 @@ watchmedo = ["PyYAML (>=3.10)"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8.1,<4.0"
-content-hash = "446d53423f89a3a378db9cff7ce8cb392e146e294d31e9d1bbfc23108a571097"
+content-hash = "892d39e8c2a50b049e3c2bb8f39b8a2126273d9644944456bd9900f6d15c5cb2"

libs/partners/ai21/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 python = ">=3.8.1,<4.0"
 langchain-core = "^0.1.28"
 langchain-text-splitters = "^0.0.1"
-ai21 = "^2.1.2"
+ai21 = "^2.2.1"
 
 [tool.poetry.group.test]
 optional = true

libs/partners/ai21/tests/integration_tests/test_chat_models.py

@@ -1,39 +1,80 @@
 """Test ChatAI21 chat model."""
-
+import pytest
 from langchain_core.messages import HumanMessage
 from langchain_core.outputs import ChatGeneration
 
 from langchain_ai21.chat_models import ChatAI21
-
-_MODEL_NAME = "j2-ultra"
+from tests.unit_tests.conftest import J2_CHAT_MODEL_NAME, JAMBA_CHAT_MODEL_NAME
 
 
-def test_invoke() -> None:
+@pytest.mark.parametrize(
+    ids=[
+        "when_j2_model",
+        "when_jamba_model",
+    ],
+    argnames=["model"],
+    argvalues=[
+        (J2_CHAT_MODEL_NAME,),
+        (JAMBA_CHAT_MODEL_NAME,),
+    ],
+)
+def test_invoke(model: str) -> None:
     """Test invoke tokens from AI21."""
-    llm = ChatAI21(model=_MODEL_NAME)
+    llm = ChatAI21(model=model)
 
     result = llm.invoke("I'm Pickle Rick", config=dict(tags=["foo"]))
     assert isinstance(result.content, str)
 
 
-def test_generation() -> None:
-    """Test invoke tokens from AI21."""
-    llm = ChatAI21(model=_MODEL_NAME)
-    message = HumanMessage(content="Hello")
+@pytest.mark.parametrize(
+    ids=[
+        "when_j2_model_num_results_is_1",
+        "when_j2_model_num_results_is_3",
+        "when_jamba_model_n_is_1",
+        "when_jamba_model_n_is_3",
+    ],
+    argnames=["model", "num_results"],
+    argvalues=[
+        (J2_CHAT_MODEL_NAME, 1),
+        (J2_CHAT_MODEL_NAME, 3),
+        (JAMBA_CHAT_MODEL_NAME, 1),
+        (JAMBA_CHAT_MODEL_NAME, 3),
+    ],
+)
+def test_generation(model: str, num_results: int) -> None:
+    """Test generation with multiple models and different result counts."""
+    # Determine the configuration key based on the model type
+    config_key = "n" if model == JAMBA_CHAT_MODEL_NAME else "num_results"
 
-    result = llm.generate([[message], [message]], config=dict(tags=["foo"]))
+    # Create the model instance using the appropriate key for the result count
+    llm = ChatAI21(model=model, **{config_key: num_results})
+
+    message = HumanMessage(content="Hello, this is a test. Can you help me please?")
+
+    result = llm.generate([[message]], config=dict(tags=["foo"]))
 
     for generations in result.generations:
-        assert len(generations) == 1
+        assert len(generations) == num_results
         for generation in generations:
             assert isinstance(generation, ChatGeneration)
             assert isinstance(generation.text, str)
             assert generation.text == generation.message.content
 
 
-async def test_ageneration() -> None:
+@pytest.mark.parametrize(
+    ids=[
+        "when_j2_model",
+        "when_jamba_model",
+    ],
+    argnames=["model"],
+    argvalues=[
+        (J2_CHAT_MODEL_NAME,),
+        (JAMBA_CHAT_MODEL_NAME,),
+    ],
+)
+async def test_ageneration(model: str) -> None:
     """Test invoke tokens from AI21."""
-    llm = ChatAI21(model=_MODEL_NAME)
+    llm = ChatAI21(model=model)
     message = HumanMessage(content="Hello")
 
     result = await llm.agenerate([[message], [message]], config=dict(tags=["foo"]))

libs/partners/ai21/tests/integration_tests/test_standard.py

@@ -1,5 +1,6 @@
 """Standard LangChain interface tests"""
 
+import time
 from typing import Type
 
 import pytest
@@ -9,17 +10,53 @@ from langchain_standard_tests.integration_tests import ChatModelIntegrationTests
 from langchain_ai21 import ChatAI21
 
 
-class TestAI21Standard(ChatModelIntegrationTests):
+class TestAI21J2(ChatModelIntegrationTests):
+    def teardown(self) -> None:
+        # avoid getting rate limited
+        time.sleep(1)
+
     @pytest.fixture
     def chat_model_class(self) -> Type[BaseChatModel]:
         return ChatAI21
 
+    @pytest.mark.xfail(reason="Emits AIMessage instead of AIMessageChunk.")
+    def test_stream(
+        self,
+        chat_model_class: Type[BaseChatModel],
+        chat_model_params: dict,
+    ) -> None:
+        super().test_stream(
+            chat_model_class,
+            chat_model_params,
+        )
+
+    @pytest.mark.xfail(reason="Emits AIMessage instead of AIMessageChunk.")
+    async def test_astream(
+        self,
+        chat_model_class: Type[BaseChatModel],
+        chat_model_params: dict,
+    ) -> None:
+        await super().test_astream(
+            chat_model_class,
+            chat_model_params,
+        )
+
     @pytest.fixture
     def chat_model_params(self) -> dict:
         return {
             "model": "j2-ultra",
         }
 
+
+class TestAI21Jamba(ChatModelIntegrationTests):
+    def teardown(self) -> None:
+        # avoid getting rate limited
+        time.sleep(1)
+
+    @pytest.fixture
+    def chat_model_class(self) -> Type[BaseChatModel]:
+        return ChatAI21
+
     @pytest.mark.xfail(reason="Emits AIMessage instead of AIMessageChunk.")
     def test_stream(
         self,
@@ -41,3 +78,9 @@ class TestAI21Standard(ChatModelIntegrationTests):
             chat_model_class,
             chat_model_params,
         )
+
+    @pytest.fixture
+    def chat_model_params(self) -> dict:
+        return {
+            "model": "jamba-instruct-preview",
+        }

libs/partners/ai21/tests/unit_tests/chat/conftest.py

@@ -0,0 +1,9 @@
+import pytest
+
+from langchain_ai21.chat.chat_adapter import ChatAdapter
+from langchain_ai21.chat.chat_factory import create_chat_adapter
+
+
+@pytest.fixture
+def chat_adapter(model: str) -> ChatAdapter:
+    return create_chat_adapter(model)

libs/partners/ai21/tests/unit_tests/chat/test_chat_adapter.py

@@ -0,0 +1,195 @@
+from typing import List
+
+import pytest
+from ai21.models import ChatMessage as J2ChatMessage
+from ai21.models import RoleType
+from ai21.models.chat import ChatMessage
+from langchain_core.messages import (
+    AIMessage,
+    BaseMessage,
+    HumanMessage,
+    SystemMessage,
+)
+from langchain_core.messages import (
+    ChatMessage as LangChainChatMessage,
+)
+
+from langchain_ai21.chat.chat_adapter import ChatAdapter
+
+_J2_MODEL_NAME = "j2-ultra"
+_JAMBA_MODEL_NAME = "jamba-instruct-preview"
+
+
+@pytest.mark.parametrize(
+    ids=[
+        "when_human_message_j2_model",
+        "when_ai_message_j2_model",
+        "when_human_message_jamba_model",
+        "when_ai_message_jamba_model",
+    ],
+    argnames=["model", "message", "expected_ai21_message"],
+    argvalues=[
+        (
+            _J2_MODEL_NAME,
+            HumanMessage(content="Human Message Content"),
+            J2ChatMessage(role=RoleType.USER, text="Human Message Content"),
+        ),
+        (
+            _J2_MODEL_NAME,
+            AIMessage(content="AI Message Content"),
+            J2ChatMessage(role=RoleType.ASSISTANT, text="AI Message Content"),
+        ),
+        (
+            _JAMBA_MODEL_NAME,
+            HumanMessage(content="Human Message Content"),
+            ChatMessage(role=RoleType.USER, content="Human Message Content"),
+        ),
+        (
+            _JAMBA_MODEL_NAME,
+            AIMessage(content="AI Message Content"),
+            ChatMessage(role=RoleType.ASSISTANT, content="AI Message Content"),
+        ),
+    ],
+)
+def test_convert_message_to_ai21_message(
+    message: BaseMessage,
+    expected_ai21_message: ChatMessage,
+    chat_adapter: ChatAdapter,
+) -> None:
+    ai21_message = chat_adapter._convert_message_to_ai21_message(message)
+    assert ai21_message == expected_ai21_message
+
+
+@pytest.mark.parametrize(
+    ids=[
+        "when_system_message_j2_model",
+        "when_langchain_chat_message_j2_model",
+    ],
+    argnames=["model", "message"],
+    argvalues=[
+        (
+            _J2_MODEL_NAME,
+            SystemMessage(content="System Message Content"),
+        ),
+        (
+            _J2_MODEL_NAME,
+            LangChainChatMessage(content="Chat Message Content", role="human"),
+        ),
+    ],
+)
+def test_convert_message_to_ai21_message__when_invalid_role__should_raise_exception(
+    message: BaseMessage,
+    chat_adapter: ChatAdapter,
+) -> None:
+    with pytest.raises(ValueError) as e:
+        chat_adapter._convert_message_to_ai21_message(message)
+    assert e.value.args[0] == (
+        f"Could not resolve role type from message {message}. "
+        f"Only support {HumanMessage.__name__} and {AIMessage.__name__}."
+    )
+
+
+@pytest.mark.parametrize(
+    ids=[
+        "when_all_messages_are_human_messages__should_return_system_none_j2_model",
+        "when_first_message_is_system__should_return_system_j2_model",
+        "when_all_messages_are_human_messages__should_return_system_none_jamba_model",
+        "when_first_message_is_system__should_return_system_jamba_model",
+    ],
+    argnames=["model", "messages", "expected_messages"],
+    argvalues=[
+        (
+            _J2_MODEL_NAME,
+            [
+                HumanMessage(content="Human Message Content 1"),
+                HumanMessage(content="Human Message Content 2"),
+            ],
+            {
+                "system": "",
+                "messages": [
+                    J2ChatMessage(
+                        role=RoleType.USER,
+                        text="Human Message Content 1",
+                    ),
+                    J2ChatMessage(
+                        role=RoleType.USER,
+                        text="Human Message Content 2",
+                    ),
+                ],
+            },
+        ),
+        (
+            _J2_MODEL_NAME,
+            [
+                SystemMessage(content="System Message Content 1"),
+                HumanMessage(content="Human Message Content 1"),
+            ],
+            {
+                "system": "System Message Content 1",
+                "messages": [
+                    J2ChatMessage(
+                        role=RoleType.USER,
+                        text="Human Message Content 1",
+                    ),
+                ],
+            },
+        ),
+        (
+            _JAMBA_MODEL_NAME,
+            [
+                HumanMessage(content="Human Message Content 1"),
+                HumanMessage(content="Human Message Content 2"),
+            ],
+            {
+                "messages": [
+                    ChatMessage(
+                        role=RoleType.USER,
+                        content="Human Message Content 1",
+                    ),
+                    ChatMessage(
+                        role=RoleType.USER,
+                        content="Human Message Content 2",
+                    ),
+                ]
+            },
+        ),
+        (
+            _JAMBA_MODEL_NAME,
+            [
+                SystemMessage(content="System Message Content 1"),
+                HumanMessage(content="Human Message Content 1"),
+            ],
+            {
+                "messages": [
+                    ChatMessage(role="system", content="System Message Content 1"),
+                    ChatMessage(role="user", content="Human Message Content 1"),
+                ],
+            },
+        ),
+    ],
+)
+def test_convert_messages(
+    chat_adapter: ChatAdapter,
+    messages: List[BaseMessage],
+    expected_messages: List[ChatMessage],
+) -> None:
+    converted_messages = chat_adapter.convert_messages(messages)
+    assert converted_messages == expected_messages
+
+
+@pytest.mark.parametrize(
+    ids=[
+        "when_j2_model",
+    ],
+    argnames=["model"],
+    argvalues=[
+        (_J2_MODEL_NAME,),
+    ],
+)
+def test_convert_messages__when_system_is_not_first(chat_adapter: ChatAdapter) -> None:
+    messages = [
+        HumanMessage(content="Human Message Content 1"),
+        SystemMessage(content="System Message Content 1"),
+    ]
+    with pytest.raises(ValueError):
+        chat_adapter.convert_messages(messages)

libs/partners/ai21/tests/unit_tests/chat/test_chat_adapter_factory.py

@@ -0,0 +1,34 @@
+from typing import Type
+
+import pytest
+
+from langchain_ai21.chat.chat_adapter import (
+    ChatAdapter,
+    J2ChatAdapter,
+    JambaChatCompletionsAdapter,
+)
+from langchain_ai21.chat.chat_factory import create_chat_adapter
+from tests.unit_tests.conftest import J2_CHAT_MODEL_NAME, JAMBA_CHAT_MODEL_NAME
+
+
+@pytest.mark.parametrize(
+    ids=[
+        "when_j2_model",
+        "when_jamba_model",
+    ],
+    argnames=["model", "expected_chat_type"],
+    argvalues=[
+        (J2_CHAT_MODEL_NAME, J2ChatAdapter),
+        (JAMBA_CHAT_MODEL_NAME, JambaChatCompletionsAdapter),
+    ],
+)
+def test_create_chat_adapter_with_supported_models(
+    model: str, expected_chat_type: Type[ChatAdapter]
+) -> None:
+    adapter = create_chat_adapter(model)
+    assert isinstance(adapter, expected_chat_type)
+
+
+def test_create_chat_adapter__when_model_not_supported() -> None:
+    with pytest.raises(ValueError):
+        create_chat_adapter("unsupported-model")

libs/partners/ai21/tests/unit_tests/conftest.py

@@ -21,6 +21,8 @@ from ai21.models import (
 from ai21.models.responses.segmentation_response import Segment
 from pytest_mock import MockerFixture
 
+J2_CHAT_MODEL_NAME = "j2-ultra"
+JAMBA_CHAT_MODEL_NAME = "jamba-instruct-preview"
 DUMMY_API_KEY = "test_api_key"
 
 BASIC_EXAMPLE_LLM_PARAMETERS = {
@@ -39,6 +41,23 @@ BASIC_EXAMPLE_LLM_PARAMETERS = {
     ),
 }
 
+BASIC_EXAMPLE_CHAT_PARAMETERS = {
+    "num_results": 3,
+    "max_tokens": 20,
+    "min_tokens": 10,
+    "temperature": 0.5,
+    "top_p": 0.5,
+    "top_k_return": 0,
+    "frequency_penalty": Penalty(scale=0.2, apply_to_numbers=True),
+    "presence_penalty": Penalty(scale=0.2, apply_to_stopwords=True),
+    "count_penalty": Penalty(
+        scale=0.2,
+        apply_to_punctuation=True,
+        apply_to_emojis=True,
+    ),
+    "n": 3,
+}
+
 SEGMENTS = [
     Segment(
         segment_type="normal_text",
@@ -82,6 +101,23 @@ BASIC_EXAMPLE_LLM_PARAMETERS_AS_DICT = {
     ).to_dict(),
 }
 
+BASIC_EXAMPLE_CHAT_PARAMETERS_AS_DICT = {
+    "num_results": 3,
+    "max_tokens": 20,
+    "min_tokens": 10,
+    "temperature": 0.5,
+    "top_p": 0.5,
+    "top_k_return": 0,
+    "frequency_penalty": Penalty(scale=0.2, apply_to_numbers=True).to_dict(),
+    "presence_penalty": Penalty(scale=0.2, apply_to_stopwords=True).to_dict(),
+    "count_penalty": Penalty(
+        scale=0.2,
+        apply_to_punctuation=True,
+        apply_to_emojis=True,
+    ).to_dict(),
+    "n": 3,
+}
+
 
 @pytest.fixture
 def mocked_completion_response(mocker: MockerFixture) -> Mock:

libs/partners/ai21/tests/unit_tests/test_chat_models.py

@@ -1,5 +1,5 @@
 """Test chat model integration."""
-from typing import List, Optional, cast
+from typing import cast
 from unittest.mock import Mock, call
 
 import pytest
@@ -7,24 +7,18 @@ from ai21 import MissingApiKeyError
 from ai21.models import ChatMessage, Penalty, RoleType
 from langchain_core.messages import (
     AIMessage,
-    BaseMessage,
     HumanMessage,
     SystemMessage,
 )
-from langchain_core.messages import (
-    ChatMessage as LangChainChatMessage,
-)
 from langchain_core.pydantic_v1 import SecretStr
 from pytest import CaptureFixture, MonkeyPatch
 
 from langchain_ai21.chat_models import (
     ChatAI21,
-    _convert_message_to_ai21_message,
-    _convert_messages_to_ai21_messages,
 )
 from tests.unit_tests.conftest import (
-    BASIC_EXAMPLE_LLM_PARAMETERS,
-    BASIC_EXAMPLE_LLM_PARAMETERS_AS_DICT,
+    BASIC_EXAMPLE_CHAT_PARAMETERS,
+    BASIC_EXAMPLE_CHAT_PARAMETERS_AS_DICT,
     DUMMY_API_KEY,
     temporarily_unset_api_key,
 )
@@ -43,7 +37,7 @@ def test_initialization__when_default_parameters_in_init() -> None:
 
 
 def test_initialization__when_custom_parameters_in_init() -> None:
-    model = "j2-mid"
+    model = "j2-ultra"
     num_results = 1
     max_tokens = 10
     min_tokens = 20
@@ -79,101 +73,6 @@ def test_initialization__when_custom_parameters_in_init() -> None:
     assert count_penalty == count_penalty
 
 
-@pytest.mark.parametrize(
-    ids=[
-        "when_human_message",
-        "when_ai_message",
-    ],
-    argnames=["message", "expected_ai21_message"],
-    argvalues=[
-        (
-            HumanMessage(content="Human Message Content"),
-            ChatMessage(role=RoleType.USER, text="Human Message Content"),
-        ),
-        (
-            AIMessage(content="AI Message Content"),
-            ChatMessage(role=RoleType.ASSISTANT, text="AI Message Content"),
-        ),
-    ],
-)
-def test_convert_message_to_ai21_message(
-    message: BaseMessage, expected_ai21_message: ChatMessage
-) -> None:
-    ai21_message = _convert_message_to_ai21_message(message)
-    assert ai21_message == expected_ai21_message
-
-
-@pytest.mark.parametrize(
-    ids=[
-        "when_system_message",
-        "when_langchain_chat_message",
-    ],
-    argnames=["message"],
-    argvalues=[
-        (SystemMessage(content="System Message Content"),),
-        (LangChainChatMessage(content="Chat Message Content", role="human"),),
-    ],
-)
-def test_convert_message_to_ai21_message__when_invalid_role__should_raise_exception(
-    message: BaseMessage,
-) -> None:
-    with pytest.raises(ValueError) as e:
-        _convert_message_to_ai21_message(message)
-    assert e.value.args[0] == (
-        f"Could not resolve role type from message {message}. "
-        f"Only support {HumanMessage.__name__} and {AIMessage.__name__}."
-    )
-
-
-@pytest.mark.parametrize(
-    ids=[
-        "when_all_messages_are_human_messages__should_return_system_none",
-        "when_first_message_is_system__should_return_system",
-    ],
-    argnames=["messages", "expected_system", "expected_messages"],
-    argvalues=[
-        (
-            [
-                HumanMessage(content="Human Message Content 1"),
-                HumanMessage(content="Human Message Content 2"),
-            ],
-            None,
-            [
-                ChatMessage(role=RoleType.USER, text="Human Message Content 1"),
-                ChatMessage(role=RoleType.USER, text="Human Message Content 2"),
-            ],
-        ),
-        (
-            [
-                SystemMessage(content="System Message Content 1"),
-                HumanMessage(content="Human Message Content 1"),
-            ],
-            "System Message Content 1",
-            [
-                ChatMessage(role=RoleType.USER, text="Human Message Content 1"),
-            ],
-        ),
-    ],
-)
-def test_convert_messages(
-    messages: List[BaseMessage],
-    expected_system: Optional[str],
-    expected_messages: List[ChatMessage],
-) -> None:
-    system, ai21_messages = _convert_messages_to_ai21_messages(messages)
-    assert ai21_messages == expected_messages
-    assert system == expected_system
-
-
-def test_convert_messages_when_system_is_not_first__should_raise_value_error() -> None:
-    messages = [
-        HumanMessage(content="Human Message Content 1"),
-        SystemMessage(content="System Message Content 1"),
-    ]
-    with pytest.raises(ValueError):
-        _convert_messages_to_ai21_messages(messages)
-
-
 def test_invoke(mock_client_with_chat: Mock) -> None:
     chat_input = "I'm Pickle Rick"
 
@@ -181,7 +80,7 @@ def test_invoke(mock_client_with_chat: Mock) -> None:
         model="j2-ultra",
         api_key=DUMMY_API_KEY,
         client=mock_client_with_chat,
-        **BASIC_EXAMPLE_LLM_PARAMETERS,
+        **BASIC_EXAMPLE_CHAT_PARAMETERS,
     )
     llm.invoke(input=chat_input, config=dict(tags=["foo"]), stop=["\n"])
 
@@ -190,7 +89,7 @@ def test_invoke(mock_client_with_chat: Mock) -> None:
         messages=[ChatMessage(role=RoleType.USER, text=chat_input)],
         system="",
         stop_sequences=["\n"],
-        **BASIC_EXAMPLE_LLM_PARAMETERS_AS_DICT,
+        **BASIC_EXAMPLE_CHAT_PARAMETERS_AS_DICT,
     )
 
 
@@ -207,7 +106,7 @@ def test_generate(mock_client_with_chat: Mock) -> None:
     llm = ChatAI21(
         model="j2-ultra",
         client=mock_client_with_chat,
-        **BASIC_EXAMPLE_LLM_PARAMETERS,
+        **BASIC_EXAMPLE_CHAT_PARAMETERS,
     )
 
     llm.generate(messages=[messages0, messages1])
@@ -226,7 +125,7 @@ def test_generate(mock_client_with_chat: Mock) -> None:
                     ChatMessage(role=RoleType.USER, text=str(messages0[2].content)),
                 ],
                 system="",
-                **BASIC_EXAMPLE_LLM_PARAMETERS_AS_DICT,
+                **BASIC_EXAMPLE_CHAT_PARAMETERS_AS_DICT,
             ),
             call(
                 model="j2-ultra",
@@ -234,7 +133,7 @@ def test_generate(mock_client_with_chat: Mock) -> None:
                     ChatMessage(role=RoleType.USER, text=str(messages1[1].content)),
                 ],
                 system="system message",
-                **BASIC_EXAMPLE_LLM_PARAMETERS_AS_DICT,
+                **BASIC_EXAMPLE_CHAT_PARAMETERS_AS_DICT,
             ),
         ]
     )

libs/partners/ai21/tests/unit_tests/test_standard.py

@@ -9,7 +9,7 @@ from langchain_standard_tests.unit_tests import ChatModelUnitTests
 from langchain_ai21 import ChatAI21
 
 
-class TestAI21Standard(ChatModelUnitTests):
+class TestAI21J2(ChatModelUnitTests):
     @pytest.fixture
     def chat_model_class(self) -> Type[BaseChatModel]:
         return ChatAI21
@@ -20,3 +20,16 @@ class TestAI21Standard(ChatModelUnitTests):
             "model": "j2-ultra",
             "api_key": "test_api_key",
         }
+
+
+class TestAI21Jamba(ChatModelUnitTests):
+    @pytest.fixture
+    def chat_model_class(self) -> Type[BaseChatModel]:
+        return ChatAI21
+
+    @pytest.fixture
+    def chat_model_params(self) -> dict:
+        return {
+            "model": "jamba-instruct",
+            "api_key": "test_api_key",
+        }