From 14e4d30659db86fa02a3dfa7a9d9787be47276d3 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Tue, 18 Apr 2023 16:32:29 +0200 Subject: [PATCH] Comet ml updates 17 04 2023 (#3074) I made a couple of improvements to the Comet tracker: * The Comet project name is configurable in various ways (code, environment variable or file), having a default value in code meant that users couldn't set the project name in an environment variable or in a file. * I added error catching when the `flush_tracker` is called in order to avoid crashing the whole process. Instead we are gonna display a warning or error log message (`extra={"show_traceback": True}` is an internal convention to force the display of the traceback when using our own logger). I decided to add the error catching after seeing the following error in the third example of the notebook: ``` COMET ERROR: Failed to export agent or LLM to Comet Traceback (most recent call last): File "/home/lothiraldan/project/cometml/langchain/langchain/callbacks/comet_ml_callback.py", line 484, in _log_model langchain_asset.save(langchain_asset_path) File "/home/lothiraldan/project/cometml/langchain/langchain/agents/agent.py", line 591, in save raise ValueError( ValueError: Saving not supported for agent executors. If you are trying to save the agent, please use the `.save_agent(...)` During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/lothiraldan/project/cometml/langchain/langchain/callbacks/comet_ml_callback.py", line 449, in flush_tracker self._log_model(langchain_asset) File "/home/lothiraldan/project/cometml/langchain/langchain/callbacks/comet_ml_callback.py", line 488, in _log_model langchain_asset.save_agent(langchain_asset_path) File "/home/lothiraldan/project/cometml/langchain/langchain/agents/agent.py", line 599, in save_agent return self.agent.save(file_path) File "/home/lothiraldan/project/cometml/langchain/langchain/agents/agent.py", line 145, in save agent_dict = self.dict() File "/home/lothiraldan/project/cometml/langchain/langchain/agents/agent.py", line 119, in dict _dict = super().dict() File "pydantic/main.py", line 449, in pydantic.main.BaseModel.dict File "pydantic/main.py", line 868, in _iter File "pydantic/main.py", line 743, in pydantic.main.BaseModel._get_value File "/home/lothiraldan/project/cometml/langchain/langchain/schema.py", line 381, in dict output_parser_dict["_type"] = self._type File "/home/lothiraldan/project/cometml/langchain/langchain/schema.py", line 376, in _type raise NotImplementedError NotImplementedError ``` I still need to investigate and try to fix it, it looks related to saving an agent to a file. --- langchain/callbacks/comet_ml_callback.py | 73 ++++++++++++++++-------- 1 file changed, 48 insertions(+), 25 deletions(-) diff --git a/langchain/callbacks/comet_ml_callback.py b/langchain/callbacks/comet_ml_callback.py index f057cea8..7917d26a 100644 --- a/langchain/callbacks/comet_ml_callback.py +++ b/langchain/callbacks/comet_ml_callback.py @@ -96,7 +96,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): self, task_type: Optional[str] = "inference", workspace: Optional[str] = None, - project_name: Optional[str] = "comet-langchain-demo", + project_name: Optional[str] = None, tags: Optional[Sequence] = None, name: Optional[str] = None, visualizations: Optional[List[str]] = None, @@ -106,7 +106,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): ) -> None: """Initialize callback handler.""" - comet_ml = import_comet_ml() + self.comet_ml = import_comet_ml() super().__init__() self.task_type = task_type @@ -133,7 +133,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): "https://github.com/comet-ml/issue_tracking/issues with the tag " "`langchain`." ) - comet_ml.LOGGER.warning(warning) + self.comet_ml.LOGGER.warning(warning) self.callback_columns: list = [] self.action_records: list = [] @@ -242,8 +242,6 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): resp.update(flatten_dict(serialized)) resp.update(self.get_custom_callback_meta()) - comet_ml = import_comet_ml() - for chain_input_key, chain_input_val in inputs.items(): if isinstance(chain_input_val, str): input_resp = deepcopy(resp) @@ -253,7 +251,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): self.action_records.append(input_resp) else: - comet_ml.LOGGER.warning( + self.comet_ml.LOGGER.warning( f"Unexpected data format provided! " f"Input Value for {chain_input_key} will not be logged" ) @@ -268,8 +266,6 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): resp.update({"action": "on_chain_end"}) resp.update(self.get_custom_callback_meta()) - comet_ml = import_comet_ml() - for chain_output_key, chain_output_val in outputs.items(): if isinstance(chain_output_val, str): output_resp = deepcopy(resp) @@ -278,7 +274,7 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): output_resp.update({chain_output_key: chain_output_val}) self.action_records.append(output_resp) else: - comet_ml.LOGGER.warning( + self.comet_ml.LOGGER.warning( f"Unexpected data format provided! " f"Output Value for {chain_output_key} will not be logged" ) @@ -449,7 +445,14 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): self._log_session(langchain_asset) if langchain_asset: - self._log_model(langchain_asset) + try: + self._log_model(langchain_asset) + except Exception: + self.comet_ml.LOGGER.error( + "Failed to export agent or LLM to Comet", + exc_info=True, + extra={"show_traceback": True}, + ) if finish: self.experiment.end() @@ -470,8 +473,6 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): self.experiment.log_text(prompt, metadata=metadata, step=step) def _log_model(self, langchain_asset: Any) -> None: - comet_ml = import_comet_ml() - model_parameters = self._get_llm_parameters(langchain_asset) self.experiment.log_parameters(model_parameters, prefix="model") @@ -487,24 +488,45 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): langchain_asset.save_agent(langchain_asset_path) self.experiment.log_model(model_name, str(langchain_asset_path)) else: - comet_ml.LOGGER.warning( + self.comet_ml.LOGGER.error( f"{e}" " Could not save Langchain Asset " f"for {langchain_asset.__class__.__name__}" ) def _log_session(self, langchain_asset: Optional[Any] = None) -> None: - llm_session_df = self._create_session_analysis_dataframe(langchain_asset) - # Log the cleaned dataframe as a table - self.experiment.log_table("langchain-llm-session.csv", llm_session_df) - - metadata = {"langchain_version": str(langchain.__version__)} - # Log the langchain low-level records as a JSON file directly - self.experiment.log_asset_data( - self.action_records, "langchain-action_records.json", metadata=metadata - ) + try: + llm_session_df = self._create_session_analysis_dataframe(langchain_asset) + # Log the cleaned dataframe as a table + self.experiment.log_table("langchain-llm-session.csv", llm_session_df) + except Exception: + self.comet_ml.LOGGER.warning( + "Failed to log session data to Comet", + exc_info=True, + extra={"show_traceback": True}, + ) - self._log_visualizations(llm_session_df) + try: + metadata = {"langchain_version": str(langchain.__version__)} + # Log the langchain low-level records as a JSON file directly + self.experiment.log_asset_data( + self.action_records, "langchain-action_records.json", metadata=metadata + ) + except Exception: + self.comet_ml.LOGGER.warning( + "Failed to log session data to Comet", + exc_info=True, + extra={"show_traceback": True}, + ) + + try: + self._log_visualizations(llm_session_df) + except Exception: + self.comet_ml.LOGGER.warning( + "Failed to log visualizations to Comet", + exc_info=True, + extra={"show_traceback": True}, + ) def _log_text_metrics(self, metrics: Sequence[dict], step: int) -> None: if not metrics: @@ -519,7 +541,6 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): return spacy = import_spacy() - comet_ml = import_comet_ml() prompts = session_df["prompts"].tolist() outputs = session_df["text"].tolist() @@ -544,7 +565,9 @@ class CometCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler): step=idx, ) except Exception as e: - comet_ml.LOGGER.warning(e) + self.comet_ml.LOGGER.warning( + e, exc_info=True, extra={"show_traceback": True} + ) return