Archives/petals
2
0
Fork 0
mirror of https://github.com/bigscience-workshop/petals synced 2024-10-31 09:20:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Show tracebacks in case of empty error messages

Browse Source
This commit is contained in:
Aleksandr Borzunov 2022-11-29 07:23:04 +00:00
parent c6e1b5a8e5
commit e37b2f526a
2 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/client/inference_session.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio import asyncio
import itertools import itertools
import logging
import time import time
from typing import AsyncIterator, List, Optional from typing import AsyncIterator, List, Optional
@ -18,7 +19,6 @@ from hivemind import (
from hivemind.moe.client.remote_expert_worker import RemoteExpertWorker from hivemind.moe.client.remote_expert_worker import RemoteExpertWorker
from hivemind.p2p import StubBase from hivemind.p2p import StubBase
from hivemind.proto import runtime_pb2 from hivemind.proto import runtime_pb2
from hivemind.utils.asyncio import aiter_with_timeout
from src.client.sequence_manager import RemoteSequenceManager from src.client.sequence_manager import RemoteSequenceManager
from src.data_structures import CHAIN_DELIMITER, ModuleUID, RemoteSpanInfo, RPCInfo from src.data_structures import CHAIN_DELIMITER, ModuleUID, RemoteSpanInfo, RPCInfo
@ -300,7 +300,8 @@ class InferenceSession:
f"Caught exception when running inference from block {block_idx} " f"Caught exception when running inference from block {block_idx} "
f"(retry in {delay:.0f} sec): {repr(e)}" f"(retry in {delay:.0f} sec): {repr(e)}"
) )
logger.debug("See detailed traceback below:", exc_info=True) traceback_level = logging.DEBUG if e.message else logging.WARNING
logger.log(traceback_level, "See detailed traceback below:", exc_info=True)
time.sleep(delay) time.sleep(delay)
self._position += n_input_tokens self._position += n_input_tokens

7
src/client/sequential_autograd.py
View File

@ -3,6 +3,7 @@ A PyTorch autograd function that runs forward/backward on a sequence of remote s
""" """
import asyncio import asyncio
import itertools import itertools
import logging
from collections import deque from collections import deque
from typing import List, Optional, Sequence, Tuple from typing import List, Optional, Sequence, Tuple
@ -86,7 +87,8 @@ async def sequential_forward(
f"Caught exception when running forward from block {block_idx} " f"Caught exception when running forward from block {block_idx} "
f"(retry in {delay:.0f} sec): {repr(e)}" f"(retry in {delay:.0f} sec): {repr(e)}"
) )
logger.debug("See detailed traceback below:", exc_info=True) traceback_level = logging.DEBUG if e.message else logging.WARNING
logger.log(traceback_level, "See detailed traceback below:", exc_info=True)
await asyncio.sleep(delay) await asyncio.sleep(delay)
return outputs, intermediate_inputs, done_sequences return outputs, intermediate_inputs, done_sequences
@ -146,7 +148,8 @@ async def sequential_backward(
f"Caught exception when running backward between blocks {span.start}-{span.end} " f"Caught exception when running backward between blocks {span.start}-{span.end} "
f"(retry in {delay:.0f} sec): {repr(e)}" f"(retry in {delay:.0f} sec): {repr(e)}"
) )
logger.debug("See detailed traceback below:", exc_info=True) traceback_level = logging.DEBUG if e.message else logging.WARNING
logger.log(traceback_level, "See detailed traceback below:", exc_info=True)
await asyncio.sleep(delay) await asyncio.sleep(delay)
# For now, we do not support mixed dummy and grad prompts # For now, we do not support mixed dummy and grad prompts