|
|
|
@ -10,6 +10,7 @@ from typing import List, Optional, Sequence, Tuple
|
|
|
|
|
import torch
|
|
|
|
|
from hivemind import MSGPackSerializer
|
|
|
|
|
from hivemind.moe.client.remote_expert_worker import RemoteExpertWorker
|
|
|
|
|
from hivemind.p2p import P2PHandlerError
|
|
|
|
|
from hivemind.utils.logging import get_logger
|
|
|
|
|
|
|
|
|
|
from petals.client.remote_forward_backward import run_remote_backward, run_remote_forward
|
|
|
|
@ -93,7 +94,7 @@ async def sequential_forward(
|
|
|
|
|
sequence_manager.on_request_success(span.peer_id)
|
|
|
|
|
break
|
|
|
|
|
except Exception as e:
|
|
|
|
|
if span is not None:
|
|
|
|
|
if span is not None and not isinstance(e, P2PHandlerError):
|
|
|
|
|
sequence_manager.on_request_failure(span.peer_id)
|
|
|
|
|
delay = sequence_manager.get_retry_delay(attempt_no)
|
|
|
|
|
logger.warning(
|
|
|
|
@ -170,7 +171,7 @@ async def sequential_backward(
|
|
|
|
|
sequence_manager.on_request_success(span.peer_id)
|
|
|
|
|
break
|
|
|
|
|
except Exception as e:
|
|
|
|
|
if span is not None:
|
|
|
|
|
if span is not None and not isinstance(e, P2PHandlerError):
|
|
|
|
|
sequence_manager.on_request_failure(span.peer_id)
|
|
|
|
|
delay = sequence_manager.get_retry_delay(attempt_no)
|
|
|
|
|
logger.warning(
|
|
|
|
|