diff --git a/libs/langchain/langchain/evaluation/comparison/eval_chain.py b/libs/langchain/langchain/evaluation/comparison/eval_chain.py
index cbb1ba505a..451f4eeb70 100644
--- a/libs/langchain/langchain/evaluation/comparison/eval_chain.py
+++ b/libs/langchain/langchain/evaluation/comparison/eval_chain.py
@@ -160,7 +160,7 @@ class PairwiseStringEvalChain(PairwiseStringEvaluator, LLMEvalChain, LLMChain):
     Example:
         >>> from langchain.chat_models import ChatOpenAI
         >>> from langchain.evaluation.comparison import PairwiseStringEvalChain
-        >>> llm = ChatOpenAI(temperature=0, model_name="gpt-4")
+        >>> llm = ChatOpenAI(temperature=0, model_name="gpt-4", model_kwargs={"random_seed": 42})
         >>> chain = PairwiseStringEvalChain.from_llm(llm=llm)
         >>> result = chain.evaluate_string_pairs(
         ...     input = "What is the chemical formula for water?",
@@ -179,7 +179,7 @@ class PairwiseStringEvalChain(PairwiseStringEvaluator, LLMEvalChain, LLMChain):
         # .     " by explaining what the formula means.\\n[[B]]"
         # }
 
-    """
+    """  # noqa: E501
 
     output_key: str = "results"  #: :meta private:
     output_parser: BaseOutputParser = Field(
diff --git a/libs/langchain/langchain/evaluation/loading.py b/libs/langchain/langchain/evaluation/loading.py
index 0f1788be9e..7579cb02ef 100644
--- a/libs/langchain/langchain/evaluation/loading.py
+++ b/libs/langchain/langchain/evaluation/loading.py
@@ -130,7 +130,9 @@ def load_evaluator(
     evaluator_cls = _EVALUATOR_MAP[evaluator]
     if issubclass(evaluator_cls, LLMEvalChain):
         try:
-            llm = llm or ChatOpenAI(model="gpt-4", temperature=0)
+            llm = llm or ChatOpenAI(
+                model="gpt-4", model_kwargs={"seed": 42}, temperature=0
+            )
         except Exception as e:
             raise ValueError(
                 f"Evaluation with the {evaluator_cls} requires a "