diff --git a/langchain/agents/chat/base.py b/langchain/agents/chat/base.py
index 1ec5842f..0935362a 100644
--- a/langchain/agents/chat/base.py
+++ b/langchain/agents/chat/base.py
@@ -44,10 +44,13 @@ class ChatAgent(Agent):
     def _extract_tool_and_input(self, text: str) -> Optional[Tuple[str, str]]:
         if FINAL_ANSWER_ACTION in text:
             return "Final Answer", text.split(FINAL_ANSWER_ACTION)[-1].strip()
-        _, action, _ = text.split("```")
+        try:
+            _, action, _ = text.split("```")
+            response = json.loads(action.strip())
+            return response["action"], response["action_input"]
 
-        response = json.loads(action.strip())
-        return response["action"], response["action_input"]
+        except Exception:
+            raise ValueError(f"Could not parse LLM output: {text}")
 
     @property
     def _stop(self) -> List[str]: