mirror of
https://github.com/bigscience-workshop/petals
synced 2024-10-31 09:20:41 +00:00
ae9e71fe8e
This pull request adds an option to run Petals server on multiple local GPUs. It uses https://github.com/BlackSamorez/tensor_parallel - 8bit approximation error same as in main (mean~=2% q0.9~=5%) - TP=1, 2, 3 (see screenshots above) - forward, grad w.r.t. input and inference exact match with main with TP=1 - `>=`80% GPU utilization with 3x 1080ti, batch = 8 tokens - throughput measured with and without TP - TP on 1080Tis has near-linear speedup comparable to the benchmarks (see first message) Co-authored-by: Iaroslav Lisniak <yalisnyak@nes.ru> Co-authored-by: Andrei Panferov <andrei@blacksamorez.ru> Co-authored-by: Alexander Borzunov <borzunov.alexander@gmail.com>
25 lines
858 B
Python
25 lines
858 B
Python
import pytest
|
|
import torch
|
|
from test_utils import MODEL_NAME
|
|
|
|
from petals.client import DistributedBloomConfig
|
|
from petals.server.throughput import measure_compute_rps, measure_network_rps
|
|
|
|
|
|
@pytest.mark.forked
|
|
@pytest.mark.parametrize("tensor_parallel", [False, True])
|
|
def test_throughput_basic(tensor_parallel: bool):
|
|
config = DistributedBloomConfig.from_pretrained(MODEL_NAME)
|
|
tensor_parallel_devices = ("cpu", "cpu") if tensor_parallel else ()
|
|
compute_rps = measure_compute_rps(
|
|
config,
|
|
device=torch.device("cpu"),
|
|
dtype=torch.bfloat16,
|
|
load_in_8bit=False,
|
|
tensor_parallel_devices=tensor_parallel_devices,
|
|
n_steps=10,
|
|
)
|
|
assert isinstance(compute_rps, float) and compute_rps > 0
|
|
network_rps = measure_network_rps(config)
|
|
assert isinstance(network_rps, float) and network_rps > 0
|