mirror of
https://github.com/hwchase17/langchain
synced 2024-11-10 01:10:59 +00:00
ebc75c5ca7
Todo - [x] copy over integration tests - [x] update docs with new instructions in #15513 - [x] add linear ticket to bump core -> community, community->langchain, and core->openai deps - [ ] (optional): add `pip install langchain-openai` command to each notebook using it - [x] Update docstrings to not need `openai` install - [x] Add serialization - [x] deprecate old models Contributor steps: - [x] Add secret names to manual integrations workflow in .github/workflows/_integration_test.yml - [x] Add secrets to release workflow (for pre-release testing) in .github/workflows/_release.yml Maintainer steps (Contributors should not do these): - [x] set up pypi and test pypi projects - [x] add credential secrets to Github Actions - [ ] add package to conda-forge Functional changes to existing classes: - now relies on openai client v1 (1.6.1) via concrete dep in langchain-openai package Codebase organization - some function calling stuff moved to `langchain_core.utils.function_calling` in order to be used in both community and langchain-openai
40 lines
874 B
Python
40 lines
874 B
Python
import pytest
|
|
|
|
from langchain_openai import ChatOpenAI, OpenAI
|
|
|
|
_EXPECTED_NUM_TOKENS = {
|
|
"ada": 17,
|
|
"babbage": 17,
|
|
"curie": 17,
|
|
"davinci": 17,
|
|
"gpt-4": 12,
|
|
"gpt-4-32k": 12,
|
|
"gpt-3.5-turbo": 12,
|
|
}
|
|
|
|
_MODELS = models = [
|
|
"ada",
|
|
"babbage",
|
|
"curie",
|
|
"davinci",
|
|
]
|
|
_CHAT_MODELS = [
|
|
"gpt-4",
|
|
"gpt-4-32k",
|
|
"gpt-3.5-turbo",
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("model", _MODELS)
|
|
def test_openai_get_num_tokens(model: str) -> None:
|
|
"""Test get_tokens."""
|
|
llm = OpenAI(model=model)
|
|
assert llm.get_num_tokens("表情符号是\n🦜🔗") == _EXPECTED_NUM_TOKENS[model]
|
|
|
|
|
|
@pytest.mark.parametrize("model", _CHAT_MODELS)
|
|
def test_chat_openai_get_num_tokens(model: str) -> None:
|
|
"""Test get_tokens."""
|
|
llm = ChatOpenAI(model=model)
|
|
assert llm.get_num_tokens("表情符号是\n🦜🔗") == _EXPECTED_NUM_TOKENS[model]
|