Archives/langchain
2
0
Fork 1
mirror of https://github.com/hwchase17/langchain synced 2024-11-04 06:00:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Same Project for Eval Runs (#8781)

Browse Source
This commit is contained in:
William FH 2023-08-04 17:51:49 -07:00 committed by GitHub
parent c8f3615aa6
commit 1d68470bac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 1 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libs/langchain/langchain/callbacks/tracers/evaluation.py
View File

@ -71,7 +71,7 @@ class EvaluatorCallbackHandler(BaseTracer):
client: Optional[Client] = None, client: Optional[Client] = None,
example_id: Optional[Union[UUID, str]] = None, example_id: Optional[Union[UUID, str]] = None,
skip_unfinished: bool = True, skip_unfinished: bool = True,
project_name: Optional[str] = None, project_name: Optional[str] = "evaluators",
**kwargs: Any, **kwargs: Any,
) -> None: ) -> None:
super().__init__(**kwargs) super().__init__(**kwargs)

4
libs/langchain/langchain/smith/evaluation/runner_utils.py
View File

@ -785,14 +785,12 @@ async def _callbacks_initializer(
project_name=project_name, client=client, use_threading=False project_name=project_name, client=client, use_threading=False
) )
) )
evaluator_project_name = f"{project_name}-evaluators" if project_name else None
if run_evaluators: if run_evaluators:
callback = EvaluatorCallbackHandler( callback = EvaluatorCallbackHandler(
client=client, client=client,
evaluators=run_evaluators, evaluators=run_evaluators,
# We already have concurrency, don't want to overload the machine # We already have concurrency, don't want to overload the machine
max_workers=1, max_workers=1,
project_name=evaluator_project_name,
) )
callbacks.append(callback) callbacks.append(callback)
evaluation_handler_collector.append(callback) evaluation_handler_collector.append(callback)
@ -1081,7 +1079,6 @@ def _run_on_examples(
tracer = LangChainTracer( tracer = LangChainTracer(
project_name=project_name, client=client, use_threading=False project_name=project_name, client=client, use_threading=False
) )
evaluator_project_name = f"{project_name}-evaluators"
run_evaluators, examples = _setup_evaluation( run_evaluators, examples = _setup_evaluation(
llm_or_chain_factory, examples, evaluation, data_type llm_or_chain_factory, examples, evaluation, data_type
) )
@ -1089,7 +1086,6 @@ def _run_on_examples(
evalution_handler = EvaluatorCallbackHandler( evalution_handler = EvaluatorCallbackHandler(
evaluators=run_evaluators or [], evaluators=run_evaluators or [],
client=client, client=client,
project_name=evaluator_project_name,
) )
callbacks: List[BaseCallbackHandler] = [tracer, evalution_handler] callbacks: List[BaseCallbackHandler] = [tracer, evalution_handler]
for i, example in enumerate(examples): for i, example in enumerate(examples):