Archives
/
langchain
mirror of https://github.com/hwchase17/langchain
2
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Session to project (#6249)

Browse Source 
Sessions are being renamed to projects in the tracer
pull/6645/head
Zander Chase 1 year ago committed by GitHub
parent 9c09861946
commit b4fe7f3a09
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 708 additions and 700 deletions
Show Stats Download Patch File Download Diff File

22
langchain/callbacks/manager.py
Unescape Escape View File

@ -106,7 +106,7 @@ def wandb_tracing_enabled(
@contextmanager
def tracing_v2_enabled(
session_name: Optional[str] = None,
project_name: Optional[str] = None,
*,
example_id: Optional[Union[str, UUID]] = None,
) -> Generator[None, None, None]:
@ -120,7 +120,7 @@ def tracing_v2_enabled(
example_id = UUID(example_id)
cb = LangChainTracer(
example_id=example_id,
session_name=session_name,
project_name=project_name,
)
tracing_v2_callback_var.set(cb)
yield
@ -131,12 +131,12 @@ def tracing_v2_enabled(
def trace_as_chain_group(
group_name: str,
*,
session_name: Optional[str] = None,
project_name: Optional[str] = None,
example_id: Optional[Union[str, UUID]] = None,
) -> Generator[CallbackManager, None, None]:
"""Get a callback manager for a chain group in a context manager."""
cb = LangChainTracer(
session_name=session_name,
project_name=project_name,
example_id=example_id,
)
cm = CallbackManager.configure(
@ -152,12 +152,12 @@ def trace_as_chain_group(
async def atrace_as_chain_group(
group_name: str,
*,
session_name: Optional[str] = None,
project_name: Optional[str] = None,
example_id: Optional[Union[str, UUID]] = None,
) -> AsyncGenerator[AsyncCallbackManager, None]:
"""Get a callback manager for a chain group in a context manager."""
cb = LangChainTracer(
session_name=session_name,
project_name=project_name,
example_id=example_id,
)
cm = AsyncCallbackManager.configure(
@ -1039,10 +1039,10 @@ def _configure(
tracing_v2_enabled_ = (
env_var_is_set("LANGCHAIN_TRACING_V2") or tracer_v2 is not None
)
tracer_session = os.environ.get("LANGCHAIN_SESSION")
tracer_project = os.environ.get(
"LANGCHAIN_PROJECT", os.environ.get("LANGCHAIN_SESSION", "default")
)
debug = _get_debug()
if tracer_session is None:
tracer_session = "default"
if (
verbose
or debug
@ -1072,7 +1072,7 @@ def _configure(
callback_manager.add_handler(tracer, True)
else:
handler = LangChainTracerV1()
handler.load_session(tracer_session)
handler.load_session(tracer_project)
callback_manager.add_handler(handler, True)
if wandb_tracing_enabled_ and not any(
isinstance(handler, WandbTracer) for handler in callback_manager.handlers
@ -1090,7 +1090,7 @@ def _configure(
callback_manager.add_handler(tracer_v2, True)
else:
try:
handler = LangChainTracer(session_name=tracer_session)
handler = LangChainTracer(project_name=tracer_project)
callback_manager.add_handler(handler, True)
except Exception as e:
logger.warning(

8
langchain/callbacks/tracers/langchain.py
Unescape Escape View File

@ -45,7 +45,7 @@ class LangChainTracer(BaseTracer):
def __init__(
self,
example_id: Optional[Union[UUID, str]] = None,
session_name: Optional[str] = None,
project_name: Optional[str] = None,
client: Optional[LangChainPlusClient] = None,
**kwargs: Any,
) -> None:
@ -55,7 +55,9 @@ class LangChainTracer(BaseTracer):
self.example_id = (
UUID(example_id) if isinstance(example_id, str) else example_id
)
self.session_name = session_name or os.getenv("LANGCHAIN_SESSION", "default")
self.project_name = project_name or os.getenv(
"LANGCHAIN_PROJECT", os.getenv("LANGCHAIN_SESSION", "default")
)
# set max_workers to 1 to process tasks in order
self.executor = ThreadPoolExecutor(max_workers=1)
self.client = client or LangChainPlusClient()
@ -103,7 +105,7 @@ class LangChainTracer(BaseTracer):
extra["runtime"] = get_runtime_environment()
run_dict["extra"] = extra
try:
self.client.create_run(**run_dict, session_name=self.session_name)
self.client.create_run(**run_dict, project_name=self.project_name)
except Exception as e:
# Errors are swallowed by the thread executor so we need to log them here
log_error_once("post", e)

64
langchain/client/runner_utils.py
Unescape Escape View File

@ -237,18 +237,18 @@ async def _gather_with_concurrency(
return results
async def _tracer_initializer(session_name: Optional[str]) -> Optional[LangChainTracer]:
async def _tracer_initializer(project_name: Optional[str]) -> Optional[LangChainTracer]:
"""
Initialize a tracer to share across tasks.
Args:
session_name: The session name for the tracer.
project_name: The project name for the tracer.
Returns:
A LangChainTracer instance with an active session.
A LangChainTracer instance with an active project.
"""
if session_name:
tracer = LangChainTracer(session_name=session_name)
if project_name:
tracer = LangChainTracer(project_name=project_name)
return tracer
else:
return None
@ -260,12 +260,12 @@ async def arun_on_examples(
*,
concurrency_level: int = 5,
num_repetitions: int = 1,
session_name: Optional[str] = None,
project_name: Optional[str] = None,
verbose: bool = False,
tags: Optional[List[str]] = None,
) -> Dict[str, Any]:
"""
Run the chain on examples and store traces to the specified session name.
Run the chain on examples and store traces to the specified project name.
Args:
examples: Examples to run the model or chain over
@ -276,7 +276,7 @@ async def arun_on_examples(
num_repetitions: Number of times to run the model on each example.
This is useful when testing success rates or generating confidence
intervals.
session_name: Session name to use when tracing runs.
project_name: Project name to use when tracing runs.
verbose: Whether to print progress.
tags: Tags to add to the traces.
@ -307,7 +307,7 @@ async def arun_on_examples(
await _gather_with_concurrency(
concurrency_level,
functools.partial(_tracer_initializer, session_name),
functools.partial(_tracer_initializer, project_name),
*(functools.partial(process_example, e) for e in examples),
)
return results
@ -386,11 +386,11 @@ def run_on_examples(
llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,
*,
num_repetitions: int = 1,
session_name: Optional[str] = None,
project_name: Optional[str] = None,
verbose: bool = False,
tags: Optional[List[str]] = None,
) -> Dict[str, Any]:
"""Run the chain on examples and store traces to the specified session name.
"""Run the chain on examples and store traces to the specified project name.
Args:
examples: Examples to run model or chain over.
@ -401,14 +401,14 @@ def run_on_examples(
num_repetitions: Number of times to run the model on each example.
This is useful when testing success rates or generating confidence
intervals.
session_name: Session name to use when tracing runs.
project_name: Project name to use when tracing runs.
verbose: Whether to print progress.
tags: Tags to add to the run traces.
Returns:
A dictionary mapping example ids to the model outputs.
"""
results: Dict[str, Any] = {}
tracer = LangChainTracer(session_name=session_name) if session_name else None
tracer = LangChainTracer(project_name=project_name) if project_name else None
for i, example in enumerate(examples):
result = run_llm_or_chain(
example,
@ -425,13 +425,13 @@ def run_on_examples(
return results
def _get_session_name(
session_name: Optional[str],
def _get_project_name(
project_name: Optional[str],
llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,
dataset_name: str,
) -> str:
if session_name is not None:
return session_name
if project_name is not None:
return project_name
current_time = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
if isinstance(llm_or_chain_factory, BaseLanguageModel):
model_name = llm_or_chain_factory.__class__.__name__
@ -446,13 +446,13 @@ async def arun_on_dataset(
*,
concurrency_level: int = 5,
num_repetitions: int = 1,
session_name: Optional[str] = None,
project_name: Optional[str] = None,
verbose: bool = False,
client: Optional[LangChainPlusClient] = None,
tags: Optional[List[str]] = None,
) -> Dict[str, Any]:
"""
Run the chain on a dataset and store traces to the specified session name.
Run the chain on a dataset and store traces to the specified project name.
Args:
client: Client to use to read the dataset.
@ -464,7 +464,7 @@ async def arun_on_dataset(
num_repetitions: Number of times to run the model on each example.
This is useful when testing success rates or generating confidence
intervals.
session_name: Name of the session to store the traces in.
project_name: Name of the project to store the traces in.
Defaults to {dataset_name}-{chain class name}-{datetime}.
verbose: Whether to print progress.
client: Client to use to read the dataset. If not provided, a new
@ -472,11 +472,10 @@ async def arun_on_dataset(
tags: Tags to add to each run in the sesssion.
Returns:
A dictionary containing the run's session name and the resulting model outputs.
A dictionary containing the run's project name and the resulting model outputs.
"""
client_ = client or LangChainPlusClient()
session_name = _get_session_name(session_name, llm_or_chain_factory, dataset_name)
client_.create_session(session_name, mode="eval")
project_name = _get_project_name(project_name, llm_or_chain_factory, dataset_name)
dataset = client_.read_dataset(dataset_name=dataset_name)
examples = client_.list_examples(dataset_id=str(dataset.id))
@ -485,12 +484,12 @@ async def arun_on_dataset(
llm_or_chain_factory,
concurrency_level=concurrency_level,
num_repetitions=num_repetitions,
session_name=session_name,
project_name=project_name,
verbose=verbose,
tags=tags,
)
return {
"session_name": session_name,
"project_name": project_name,
"results": results,
}
@ -500,12 +499,12 @@ def run_on_dataset(
llm_or_chain_factory: MODEL_OR_CHAIN_FACTORY,
*,
num_repetitions: int = 1,
session_name: Optional[str] = None,
project_name: Optional[str] = None,
verbose: bool = False,
client: Optional[LangChainPlusClient] = None,
tags: Optional[List[str]] = None,
) -> Dict[str, Any]:
"""Run the chain on a dataset and store traces to the specified session name.
"""Run the chain on a dataset and store traces to the specified project name.
Args:
dataset_name: Name of the dataset to run the chain on.
@ -516,7 +515,7 @@ def run_on_dataset(
num_repetitions: Number of times to run the model on each example.
This is useful when testing success rates or generating confidence
intervals.
session_name: Name of the session to store the traces in.
project_name: Name of the project to store the traces in.
Defaults to {dataset_name}-{chain class name}-{datetime}.
verbose: Whether to print progress.
client: Client to use to access the dataset. If None, a new client
@ -524,22 +523,21 @@ def run_on_dataset(
tags: Tags to add to each run in the sesssion.
Returns:
A dictionary containing the run's session name and the resulting model outputs.
A dictionary containing the run's project name and the resulting model outputs.
"""
client_ = client or LangChainPlusClient()
session_name = _get_session_name(session_name, llm_or_chain_factory, dataset_name)
client_.create_session(session_name, mode="eval")
project_name = _get_project_name(project_name, llm_or_chain_factory, dataset_name)
dataset = client_.read_dataset(dataset_name=dataset_name)
examples = client_.list_examples(dataset_id=str(dataset.id))
results = run_on_examples(
examples,
llm_or_chain_factory,
num_repetitions=num_repetitions,
session_name=session_name,
project_name=project_name,
verbose=verbose,
tags=tags,
)
return {
"session_name": session_name,
"project_name": project_name,
"results": results,
}

1296
langchain/experimental/client/tracing_datasets.ipynb
View File

File diff suppressed because it is too large Load Diff

10
poetry.lock generated
Unescape Escape View File

@ -4362,13 +4362,13 @@ tests = ["doctest", "pytest", "pytest-mock"]
[[package]]
name = "langchainplus-sdk"
version = "0.0.15"
description = "Client library to connect to the LangChainPlus LLM Tracing and Evaluation Platform."
version = "0.0.17"
description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
optional = false
python-versions = ">=3.8.1,<4.0"
files = [
{file = "langchainplus_sdk-0.0.15-py3-none-any.whl", hash = "sha256:e69bdbc8af6007ef2f774248d2483bbaf2d75712b1acc9ea50eda3b9f6dc567d"},
{file = "langchainplus_sdk-0.0.15.tar.gz", hash = "sha256:ce40e9e3b6d42741f0a2aa89f83a12f2648f38690a9dd57e5fe3a56f2f232908"},
{file = "langchainplus_sdk-0.0.17-py3-none-any.whl", hash = "sha256:899675fe850bb0829691ce7643d5c3b4425de1535b6f2d6ce1e5f5457ffb05bf"},
{file = "langchainplus_sdk-0.0.17.tar.gz", hash = "sha256:6520c864a23dcadbe6fb7233a117347f6acc32725a97758e59354704c50de303"},
]
[package.dependencies]
@ -11771,4 +11771,4 @@ text-helpers = ["chardet"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.8.1,<4.0"
content-hash = "09d46ad12369c6a16513558618553623cd520c2855bff3b8fe8248e1b18cbb94"
content-hash = "6e495e4f58127a5d2001385404b973896e275f5ca71a6ebe856cb114977189d1"

2
pyproject.toml
Unescape Escape View File

@ -106,7 +106,7 @@ pyspark = {version = "^3.4.0", optional = true}
clarifai = {version = "9.1.0", optional = true}
tigrisdb = {version = "^1.0.0b6", optional = true}
nebula3-python = {version = "^3.4.0", optional = true}
langchainplus-sdk = ">=0.0.13"
langchainplus-sdk = ">=0.0.17"
awadb = {version = "^0.3.3", optional = true}
azure-search-documents = {version = "11.4.0a20230509004", source = "azure-sdk-dev", optional = true}
openllm = {version = ">=0.1.6", optional = true}

6
tests/unit_tests/client/test_runner_utils.py
Unescape Escape View File

@ -176,7 +176,7 @@ async def test_arun_on_dataset(monkeypatch: pytest.MonkeyPatch) -> None:
{"result": f"Result for example {example.id}"} for _ in range(n_repetitions)
]
def mock_create_session(*args: Any, **kwargs: Any) -> None:
def mock_create_project(*args: Any, **kwargs: Any) -> None:
pass
with mock.patch.object(
@ -186,7 +186,7 @@ async def test_arun_on_dataset(monkeypatch: pytest.MonkeyPatch) -> None:
), mock.patch(
"langchain.client.runner_utils._arun_llm_or_chain", new=mock_arun_chain
), mock.patch.object(
LangChainPlusClient, "create_session", new=mock_create_session
LangChainPlusClient, "create_project", new=mock_create_project
):
client = LangChainPlusClient(api_url="http://localhost:1984", api_key="123")
chain = mock.MagicMock()
@ -195,7 +195,7 @@ async def test_arun_on_dataset(monkeypatch: pytest.MonkeyPatch) -> None:
dataset_name="test",
llm_or_chain_factory=lambda: chain,
concurrency_level=2,
session_name="test_session",
project_name="test_project",
num_repetitions=num_repetitions,
client=client,
)

Write Preview
Loading…
Cancel
Save