Repository navigation
Expand file tree
/
Copy pathcommunication.py
More file actions
859 lines (772 loc) · 41.6 KB
/
Copy pathcommunication.py
File metadata and controls
859 lines (772 loc) · 41.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
import functools
import math
import socket
from collections.abc import Sequence
from dataclasses import dataclass, field
from datetime import timedelta
from typing import Any, NamedTuple
from urllib.parse import urlparse
import torch
import torch.distributed as dist
from torch.distributed.constants import default_pg_nccl_timeout, default_pg_timeout
from torch.distributed.distributed_c10d import _set_pg_timeout
from mstar.distributed.all_reduce import AllReduce
from mstar.distributed.base import PARALLEL_AXES
from mstar.distributed.utils import resolve_dist_timeout
def default_group_timeout(backend: str) -> timedelta:
"""The timeout ``init_process_group`` gives ``backend``'s group when none is passed.
With ``init_method`` it also sets the store to it; with ``store=`` it
leaves the store alone, so the caller passes this to both.
"""
if dist.Backend(backend) == dist.Backend.NCCL and default_pg_nccl_timeout is not None:
return default_pg_nccl_timeout
return default_pg_timeout
def rendezvous_store(init_method: str, rank: int, world_size: int, timeout: timedelta) -> dist.TCPStore:
"""The ``tcp://host:port`` store every rank meets at, listening on ``host`` only.
``init_process_group(init_method="tcp://...")`` has rank 0 serve its
TCPStore on every interface whatever the host, so rank 0 binds the
listener itself and hands it over.
"""
url = urlparse(init_method)
if url.scheme != "tcp" or not url.hostname or not url.port:
raise ValueError(f"init_method must be tcp://host:port; got {init_method!r}")
if rank != 0:
return dist.TCPStore(url.hostname, url.port, world_size, is_master=False, timeout=timeout)
listener = socket.create_server((url.hostname, url.port), backlog=max(world_size, 128))
# the store owns the descriptor from here
return dist.TCPStore(
url.hostname, url.port, world_size, is_master=True, timeout=timeout,
wait_for_workers=False, master_listen_fd=listener.detach(),
)
# The collective itself stays out of torch.compile whoever calls it: traced as a functional
# collective it gave an illegal memory access in Cosmos3's batched capture.
@torch.compiler.disable
def _all_to_all_single(
group: dist.ProcessGroup, send: torch.Tensor, recv: torch.Tensor,
recv_splits: list[int] | None = None, send_splits: list[int] | None = None,
) -> torch.Tensor:
dist.all_to_all_single(recv, send, recv_splits, send_splits, group=group)
return recv
# torch 2.13 renamed the into-tensor collectives to the *_single scheme.
# Import-time selection keeps the fallback out of compiled call paths.
_all_gather_single = getattr(dist, "all_gather_single", None)
if _all_gather_single is None:
_all_gather_single = getattr(
getattr(dist, "distributed_c10d", None), "all_gather_single", None
)
_all_gather_into_tensor = _all_gather_single or dist.all_gather_into_tensor
def _barrier(group: dist.ProcessGroup | None = None) -> None:
"""Barrier without torch 2.13's implicit-device warning on NCCL groups."""
if dist.get_backend(group) == dist.Backend.NCCL:
dist.barrier(group=group, device_ids=[torch.cuda.current_device()])
else:
dist.barrier(group=group)
def _rms_norm(r: torch.Tensor, weight: torch.Tensor, eps: float, weight_bias: float) -> torch.Tensor:
"""The fused kernel's norm, unfused: fp32, scaling by ``weight + weight_bias``."""
v = r.float()
v = v * torch.rsqrt(v.pow(2).mean(-1, keepdim=True) + eps)
return (v * (weight.float() + weight_bias)).to(r.dtype)
class ArFusionWorkspace(NamedTuple):
"""A fused all-reduce workspace of a ``CommGroup``: its id in
``ar_fusion``'s registry, for inputs of up to ``max_tokens`` rows, and
``allreduce_rmsnorm``'s ``[max_tokens, hidden]`` zero residual (shared,
which is safe only because the fused op never writes its residual)."""
ws_id: int
max_tokens: int
zeros: torch.Tensor
class CommGroup:
"""A communication group over one axis of the worker device mesh, or over
a whole lockstep instance.
An axis group runs the model's collectives: tensor parallelism
(all-reduce / all-gather of row-parallel projections) or Ulysses sequence
parallelism (all-to-all around attention). An instance group only agrees
on the host (``barrier``, ``agree``), so it has a gloo group and
no device group. ``rank`` / ``world_size``
are relative to this group; ``global_rank`` is the worker's rank in
the world.
"""
def __init__(
self,
my_global_rank: int,
my_group_rank: int,
group_members: list[int]
):
self.global_rank = my_global_rank
self.rank = my_group_rank
self.group_members = group_members
self.world_size = len(group_members)
self.device_group = None
# host channel, for rank votes (``agree``), ``barrier`` and object
# collectives (workspace rendezvous) that should not go through NCCL:
# a gloo twin of ``device_group``, or on XPU the mixed group's gloo
# side; a lockstep instance's only group
self.cpu_group = None
self.initialized = False
# (dtype, hidden) -> its fused all-reduce workspace; see ``init_allreduce_fusion``
self._ar_fusion: dict[tuple[torch.dtype, int], ArFusionWorkspace] = {}
# NCCL until ``init_all_reduce`` sets up the deployment's backend
self._all_reduce = AllReduce()
def init_all_reduce(self, mode: str | None = None, max_kb: int | None = None) -> None:
"""Set up the all-reduce backend the deployment config's ``tp_allreduce``
(``mode``) and ``tp_allreduce_max_kb`` (``max_kb``) ask for (see
``AllReduce.new``). Collective across the group — every member calls
this from ``init_dist`` at the same point."""
self._all_reduce = AllReduce.new(self, mode, max_kb)
def _device_pg(self) -> "dist.ProcessGroup":
"""The group a device collective runs on. torch reads ``group=None`` as
the whole world, so a group without one refuses instead."""
if self.device_group is None:
raise ValueError(
f"device collective on ranks {self.group_members}: a host-only (lockstep instance) "
"group, or one init_dist has not set up"
)
return self.device_group
def agree(self, values: list[int], op: dist.ReduceOp = dist.ReduceOp.MIN) -> list[int]:
"""Reduce each of ``values`` over the group's ranks with ``op``: ``MIN`` ANDs
flags, ``MAX`` takes the largest. Collective: every member calls it at the
same point. Over ``cpu_group``, so the answer costs no device sync."""
if self.world_size == 1 or not values:
return list(values)
if self.cpu_group is None:
raise ValueError(
f"agree over ranks {self.group_members} before init_dist set up their cpu_group"
)
t = torch.tensor(values, dtype=torch.int64)
dist.all_reduce(t, op=op, group=self.cpu_group)
return t.tolist()
@classmethod
def trivial(cls) -> "CommGroup":
"""A degenerate single-rank group. All collectives are no-ops;
``init_process_group`` does nothing. Useful as the default for
non-parallel runs so the same code path works everywhere."""
return cls(my_global_rank=0, my_group_rank=0, group_members=[0])
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
if self.world_size == 1:
return input_
if dim < 0:
# Convert negative dim to positive
dim += input_.dim()
input_size = input_.size()
output_size = (input_size[0] * self.world_size,) + input_size[1:]
# Allocate output tensor
output_tensor = torch.empty(
output_size, dtype=input_.dtype, device=input_.device
)
# All-gather
_all_gather_into_tensor(output_tensor, input_, group=self._device_pg())
# Reshape
output_tensor = output_tensor.reshape((self.world_size,) + input_size)
output_tensor = output_tensor.movedim(0, dim)
output_tensor = output_tensor.reshape(
input_size[:dim]
+ (self.world_size * input_size[dim],)
+ input_size[dim + 1 :]
)
return output_tensor
def barrier(self):
"""On the host: waits on the other ranks, never on a device."""
if self.world_size == 1:
return
if self.cpu_group is None:
# torch reads ``group=None`` as the whole world
raise ValueError(
f"barrier over ranks {self.group_members} before init_dist set up their cpu_group"
)
dist.barrier(group=self.cpu_group)
def all_reduce_buffer(
self, shape: tuple[int, ...], dtype: torch.dtype, device: torch.device
) -> torch.Tensor:
"""Where to write the partial that the next ``all_reduce`` sums. On
the symmetric-memory path this is the symm buffer, so ``all_reduce``
skips its copy in and returns the sum in a new tensor: use its return
value, and all-reduce nothing else on this group in between."""
return self._all_reduce.buffer(shape, dtype, device)
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
if self.world_size == 1:
return input_
return self._all_reduce.all_reduce(input_, self._device_pg())
def init_allreduce_fusion(
self, hidden: int, dtype: torch.dtype, max_tokens: int = 2048,
) -> bool:
"""Set up the fused kernel of ``allreduce_add_rmsnorm`` and
``allreduce_rmsnorm`` for ``[<=max_tokens, hidden]`` inputs of ``dtype``,
with ``allreduce_rmsnorm``'s zero residual (16 MiB per rank at 2048 x
4096, bf16). Returns whether it is available for them.
Collective: every member calls it at load, in the same order with the
same arguments. The lockstep nodes of an instance share its group, so
it uses one workspace per ``(dtype, hidden)``. A larger ``max_tokens``
builds another if it can and leaves the old one allocated, outside torch's
allocator: 512 MiB per rank in each shape measured at TP2 on H200, with
the MNNVL backend FlashInfer 0.7.2's "auto" chose. Other inputs take the
group's ``all_reduce`` and a separate norm.
"""
if self.world_size == 1:
return False
have = self._ar_fusion.get((dtype, hidden))
if have is not None and have.max_tokens >= max_tokens:
return True
from mstar.distributed.ar_fusion import create_workspace
ws_id = create_workspace(self, max_tokens, hidden, dtype)
if ws_id is None:
return False
# on the device the workspace was just built on
zeros = torch.zeros(max_tokens, hidden, dtype=dtype, device=torch.accelerator.current_accelerator())
self._ar_fusion[(dtype, hidden)] = ArFusionWorkspace(ws_id, max_tokens, zeros)
return True
def allreduce_add_rmsnorm(
self, x: torch.Tensor, residual: torch.Tensor, weight: torch.Tensor,
eps: float, weight_bias: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor]:
"""``r = all_reduce(x) + residual``; returns ``(rms_norm(r), r)``,
scaling by ``weight + weight_bias``. ``x`` is a row-parallel
projection's partial sum (its linear skipped ``reduce_results``)."""
ws = self._ar_fusion.get((x.dtype, x.shape[-1]))
if (ws is not None and x.shape[0] <= ws.max_tokens
and x.is_contiguous() and residual.is_contiguous()):
from mstar.distributed.ar_fusion import allreduce_add_rmsnorm
return allreduce_add_rmsnorm(x, residual, weight, eps, weight_bias, ws.ws_id)
r = self.all_reduce(x) + residual
return _rms_norm(r, weight, eps, weight_bias), r
def allreduce_rmsnorm(
self, x: torch.Tensor, weight: torch.Tensor, eps: float, weight_bias: float = 0.0,
) -> torch.Tensor:
"""``rms_norm(all_reduce(x))``, a norm before the residual add, in the
same fused kernel: it always adds a residual, so this adds zeros."""
ws = self._ar_fusion.get((x.dtype, x.shape[-1]))
if ws is not None and x.shape[0] <= ws.max_tokens and x.is_contiguous():
from mstar.distributed.ar_fusion import allreduce_add_rmsnorm
return allreduce_add_rmsnorm(
x, ws.zeros[: x.shape[0]], weight, eps, weight_bias, ws.ws_id,
)[0]
return _rms_norm(self.all_reduce(x), weight, eps, weight_bias)
def broadcast(self, tensor: torch.Tensor, src: int = 0) -> torch.Tensor:
"""Broadcast a tensor from source rank to all ranks."""
if self.world_size == 1:
return tensor
dist.broadcast(tensor, self.group_members[src], self._device_pg())
return tensor
def all_to_all_rows(self, x: torch.Tensor, send_rows: Sequence[int], recv_rows: Sequence[int]) -> torch.Tensor:
"""``x``'s rows (its first dim) to the group's ranks, ``send_rows[i]`` of them to group
rank ``i``, in rank order; back come the rows each rank sent this one, ``recv_rows[i]``
from rank ``i``, in rank order. One all-to-all over the whole group. The counts are
group-relative, and every member calls it with counts that pair up (what a sends b is what
b receives from a). Replays from a CUDA graph."""
if self.world_size == 1:
return x
x = x.contiguous()
row = math.prod(x.shape[1:])
recv = x.new_empty(sum(recv_rows), *x.shape[1:])
_all_to_all_single(
self._device_pg(), x.view(-1), recv.view(-1), [n * row for n in recv_rows], [n * row for n in send_rows],
)
return recv
def connect_pairs(self, device: torch.device) -> None:
"""One one-element exchange per rank offset, so every pair of ranks has exchanged once:
NCCL connects a pair on its first exchange, which for a ring's point-to-point hops would
come in its first request, after warmup and the capture plan read free memory, and a
routing fault would hang there rather than at startup. Collective: every member calls it
at the same point."""
one = torch.zeros(1, device=device)
for offset in range(1, self.world_size):
send, recv = [0] * self.world_size, [0] * self.world_size
send[(self.rank + offset) % self.world_size] = recv[(self.rank - offset) % self.world_size] = 1
self.all_to_all_rows(one, send, recv)
# Eager as a whole, the boundary where the per-rank sizes become the collective's host split
# lists: traced, its branches on them would compile per sequence length, and a traced variant
# that kept them symbolic measured no faster on Cosmos3 SP.
@torch.compiler.disable
def all_to_all(
self,
input_: torch.Tensor | Sequence[torch.Tensor],
scatter_dim: int,
gather_dim: int,
scatter_sizes: list[int] | None = None,
gather_sizes: list[int] | None = None,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
"""Redistribute ``input_`` across the group: split it into
``world_size`` pieces along ``scatter_dim`` (piece i goes to rank i)
and concatenate the pieces received from every rank along
``gather_dim``.
This is the primitive behind Ulysses sequence parallelism: with
``scatter_dim`` = heads and ``gather_dim`` = sequence it converts a
sequence-sharded ``[seq/P, heads, dim]`` tensor into a head-sharded
``[seq, heads/P, dim]`` one (and the reverse with the dims swapped).
``scatter_sizes`` / ``gather_sizes`` give per-rank extents for an
uneven split / gather (e.g. a sequence length not divisible by the
group size); when ``None`` the respective dimension is split evenly
(and ``scatter_dim`` must then be divisible by ``world_size``).
``input_`` may be several tensors of one dtype and one extent along
``gather_dim``, each split evenly (q, k and v with their own head counts):
they go in one exchange and come back as a tuple. Each exchange is one
``all_to_all_single`` over a buffer packed in a single copy, which
replays from a CUDA graph. It runs outside torch.compile, so a compiled
caller breaks around this call.
"""
tensors = (input_,) if isinstance(input_, torch.Tensor) else tuple(input_)
# packed into one buffer of the first's dtype, which would silently convert the others
assert len({x.dtype for x in tensors}) == 1, f"all_to_all packs one dtype, got {[x.dtype for x in tensors]}"
if self.world_size == 1:
return input_ if isinstance(input_, torch.Tensor) else tensors
world_size = self.world_size
# positive dims: unflatten below adds a dim, which would shift a negative one
scatter_dim %= tensors[0].dim()
gather_dim %= tensors[0].dim()
if scatter_sizes is not None and len(set(scatter_sizes)) > 1:
assert len(tensors) == 1, "all_to_all: several tensors need an even scatter"
out = self._all_to_all_uneven_scatter(tensors[0], scatter_dim, gather_dim, scatter_sizes, gather_sizes)
return out if isinstance(input_, torch.Tensor) else (out,)
# per tensor [.., N, ..] -> [P, .., N/P, ..]; rank r's block of the send buffer holds
# its piece of each tensor one after another, so each tensor unpacks contiguous
blocks = [x.unflatten(scatter_dim, (world_size, -1)).movedim(scatter_dim, 0) for x in tensors]
numels = [blk[0].numel() for blk in blocks]
send = blocks[0].new_empty(world_size, sum(numels))
for blk, piece in zip(blocks, send.split(numels, dim=1), strict=True):
piece.view(blk.shape).copy_(blk)
if gather_sizes is None or len(set(gather_sizes)) == 1:
# the block received from rank r has this rank's own block's layout
recv = _all_to_all_single(self._device_pg(), send, torch.empty_like(send))
outs = [
piece.view(blk.shape).movedim(0, gather_dim).flatten(gather_dim, gather_dim + 1)
for blk, piece in zip(blocks, recv.split(numels, dim=1), strict=True)
]
else:
# rank r's block: each tensor's piece with the gathered dim at rank r's extent
shapes = [
[blk.shape[1:gather_dim + 1] + (n,) + blk.shape[gather_dim + 2:] for blk in blocks]
for n in gather_sizes
]
rank_numels = [[int(torch.Size(shape).numel()) for shape in row] for row in shapes]
recv = _all_to_all_single(
self._device_pg(), send.flatten(), send.new_empty(sum(map(sum, rank_numels))),
[sum(row) for row in rank_numels], [sum(numels)] * world_size,
)
per_rank = [
[piece.view(shape) for piece, shape in zip(block.split(row), shape_row, strict=True)]
for block, row, shape_row in zip(recv.split([sum(row) for row in rank_numels]), rank_numels, shapes,
strict=True)
]
outs = [torch.cat([pieces[i] for pieces in per_rank], dim=gather_dim) for i in range(len(tensors))]
return outs[0] if isinstance(input_, torch.Tensor) else tuple(outs)
def _all_to_all_uneven_scatter(
self, input_: torch.Tensor, scatter_dim: int, gather_dim: int,
scatter_sizes: list[int], gather_sizes: list[int] | None,
) -> torch.Tensor:
chunks = input_.split(scatter_sizes, dim=scatter_dim)
send = torch.cat([chunk.contiguous().flatten() for chunk in chunks])
mine = list(chunks[self.rank].shape)
if gather_sizes is None:
gather_sizes = [mine[gather_dim]] * self.world_size
shapes = [mine[:gather_dim] + [n] + mine[gather_dim + 1:] for n in gather_sizes]
numels = [int(torch.Size(shape).numel()) for shape in shapes]
recv = _all_to_all_single(
self._device_pg(), send, input_.new_empty(sum(numels)),
numels, [chunk.numel() for chunk in chunks],
)
pieces = [
piece.view(shape)
for piece, shape in zip(recv.split(numels), shapes, strict=True)
]
return torch.cat(pieces, dim=gather_dim)
def sp_seq_split(total: int, world_size: int) -> list[int]:
"""Even per-rank token counts summing to ``total``, earlier ranks taking the remainder: how a
sequence splits over a group's ranks, contiguous in rank order. Arithmetic only, so a
compiled caller traces it without a graph break, a dynamic ``total`` staying symbolic."""
return [(total + world_size - 1 - r) // world_size for r in range(world_size)]
def gather_sequence(group: CommGroup, x_shard: torch.Tensor, seq_sizes: Sequence[int], dim: int = 0) -> torch.Tensor:
"""Every rank's share along ``dim`` (``seq_sizes``, per rank) back as the whole tensor, in rank
order. Each share is padded to the largest, so every rank gathers the same shape, then trimmed.
Without a branch on the sizes, so a compiled caller keeps them symbolic and compiles once for
every sequence length."""
world_size = group.world_size
if world_size == 1:
return x_shard
dim %= x_shard.dim() # positive: unflatten below adds a dim, which would shift a negative one
max_sz = functools.reduce(torch.sym_max, seq_sizes)
shape = list(x_shard.shape)
shape[dim] = max_sz
local = x_shard.new_empty(shape)
local.narrow(dim, 0, x_shard.size(dim)).copy_(x_shard)
chunks = group.all_gather(local, dim=dim).unflatten(dim, (world_size, max_sz)).unbind(dim)
return torch.cat([chunk.narrow(dim, 0, n) for chunk, n in zip(chunks, seq_sizes, strict=True)], dim=dim)
def my_rows(group: CommGroup, batch: int) -> slice:
"""This rank's even share of ``batch`` leading rows, for a comm group that
splits a batch rather than the sequence (e.g. CFG branches over the cfg
group). No communication; ``group.all_gather(x, dim=0)`` reassembles the
batch, and a trivial group keeps every row. A batch the group cannot
split evenly is refused: every rank holds the same ``batch`` and group, so
all of them raise."""
share, rem = divmod(batch, group.world_size)
if rem or not share:
raise ValueError(f"cannot split {batch} rows evenly over a {group.world_size}-rank group")
return slice(group.rank * share, (group.rank + 1) * share)
@dataclass
class JointGroups:
"""A node's comm groups on this worker: one per mesh axis, for the model's
own collectives, and ``instance``, the whole lockstep instance (its ranks
in ``WorkerGraph._instance_ranks`` order, which the conductor's instance
rank follows too). Anything every rank of the instance must agree on — a
barrier, a vote, a symmetric-state check — is one host-side collective on
``instance``."""
tp_group: CommGroup
sp_group: CommGroup
instance: CommGroup
cfg_group: CommGroup = field(default_factory=CommGroup.trivial)
@property
def rank(self):
"""This worker's rank in the node's lockstep instance; 0 leads it."""
return self.instance.rank
@property
def world_size(self):
"""Ranks in the node's lockstep instance."""
return self.instance.world_size
def broadcast(self, tensor: torch.Tensor) -> torch.Tensor:
"""Broadcast from instance rank 0 to the whole instance: down each axis
in turn, from the member at coordinate 0, which instance rank 0 is on
every axis. Each step spreads its value along one more axis, so any
order works."""
for axis in reversed(PARALLEL_AXES):
tensor = getattr(self, f"{axis}_group").broadcast(tensor)
return tensor
def agree(self, values: list[int], op: dist.ReduceOp = dist.ReduceOp.MIN) -> list[int]:
"""``CommGroup.agree`` over the whole lockstep instance: one host
all-reduce on ``instance``."""
return self.instance.agree(values, op)
@dataclass
class WorkerParallelGroups:
"""Per-worker view of the parallelism comm groups in the run.
The parallel axes (``PARALLEL_AXES``) are orthogonal axes of one device
mesh, so a node may carry one comm group of each. The whole block is the
node's lockstep *instance* — the unit the conductor routes a request to —
exposed via ``get_instance_rank_for_node`` /
``get_instance_world_size_for_node``.
"""
num_workers: int
global_rank: int
# True iff any worker in the run has a parallel axis. Set by
# GlobalParallelConfig from the global worker-graph view so all ranks
# agree.
any_parallelism: bool = False
# Every distinct axis rank tuple in the run — the union over the mesh
# axes, sorted for stable iteration order across workers. Set by
# GlobalParallelConfig. ``init_dist`` calls ``dist.new_group`` once per
# entry on every rank — including ranks that aren't members of the
# group — because PyTorch assigns an auto-incrementing tag inside
# ``new_group`` that all ranks must agree on; asymmetric call counts
# deadlock the participating ranks.
world_parallel_groups: list[tuple[int, ...]] = field(default_factory=list)
# Per-node comm groups by mesh axis (``PARALLEL_AXES``): the SP group
# all-to-alls around attention; the TP group all-reduces the row-parallel
# projections.
axis_groups: dict[str, dict[str, CommGroup]] = field(
default_factory=lambda: {axis: {} for axis in PARALLEL_AXES}
)
# Per-node lockstep instance groups (``JointGroups.instance``), gloo (on
# XPU the world's mixed group, like the XPU axis groups);
# a single-axis instance shares that axis group's.
node_to_instance_comm: dict[str, CommGroup] = field(default_factory=dict)
# Global topology metadata. Unlike the comm-group maps above, these maps
# describe every replica, including ones this worker does not host. A
# shape is the per-axis sizes, in ``PARALLEL_AXES`` order.
node_to_parallel_shapes: dict[
str, frozenset[tuple[int, ...]]
] = field(default_factory=dict)
node_to_instance_groups: dict[
str, frozenset[tuple[int, ...]]
] = field(default_factory=dict)
_device: torch.device | None = field(default=None, init=False, repr=False)
node_to_joint_group: dict[str, JointGroups] = field(default_factory=dict)
# Process-group timeout in seconds for load and warmup, from the config's
# ``dist_timeout_s``. None = torch default.
dist_timeout_s: float | None = None
# Small-message all-reduce path from the deployment config's
# ``tp_allreduce``. None = NCCL.
tp_allreduce: str | None = None
# Largest message it takes, from ``tp_allreduce_max_kb``; None = 512.
tp_allreduce_max_kb: int | None = None
def add(self, node: str, comm_group: CommGroup, axis: str):
# disallow colocation of multiple comm groups on the same node and axis
per_node = self.axis_groups[axis]
if node in per_node and per_node[node].group_members != comm_group.group_members:
raise ValueError(
f"Node {node} already has a {axis} comm group assigned for worker {self.global_rank}"
)
per_node.setdefault(node, comm_group)
def init_dist(
self, init_method="tcp://127.0.0.1:29500",
device: torch.device | None = None,
):
"""Initialize the NCCL world group and per-node parallel subgroups.
Every worker calls ``dist.init_process_group`` when *any* worker
in the run has a parallel axis (``self.any_parallelism``) —
otherwise ranks with no local parallelism would skip the call and
the participating ranks would hang waiting for them.
Subgroup creation: PyTorch's ``dist.new_group`` is collective on
the global world. It assigns an auto-incrementing tag inside the
call that every rank must agree on; if non-member ranks skip the
call, the tag counter drifts and member ranks deadlock. We
therefore call ``new_group`` once per distinct rank tuple on
every rank — members keep the returned handle, non-members
discard it.
"""
device = device or torch.device("cuda", self.global_rank)
self._device = device
if device.type != "cpu":
torch.accelerator.set_device_index(device)
backend = dist.get_default_backend_for_device(device.type)
if not self.any_parallelism:
return
if device.type == "xpu":
backend = f"cpu:gloo,xpu:{backend}"
timeout_kwargs = resolve_dist_timeout(self.dist_timeout_s)
# a configured dist_timeout_s covers the rendezvous store too
timeout = timeout_kwargs.get("timeout", default_group_timeout(backend))
dist.init_process_group(
backend=backend,
store=rendezvous_store(init_method, self.global_rank, self.num_workers, timeout),
world_size=self.num_workers,
rank=self.global_rank,
device_id=device,
timeout=timeout,
)
# Gloo groups, members only: one per rank tuple of a mesh axis, and one
# per multi-axis lockstep instance, which agrees only on the host and
# so gets no NCCL communicator (a single-axis instance shares its
# axis's). With the default group bound to a device, a non-member's
# call would join an ncclCommSplit that members never make for a gloo
# group. torch names a members-only group after its ranks and how many
# groups this process holds, so every rank makes them before the device
# groups (registered only on their members, they would skew the count),
# in one global order, and a rank outside a tuple makes a group of just
# itself: the count, and so the name the members rendezvous under, is
# then equal on every member. On XPU an axis's device group doubles as
# its cpu group, and an instance gets the world's mixed group, made on
# every rank.
axis_tuples = set(self.world_parallel_groups)
instance_tuples = {
block for blocks in self.node_to_instance_groups.values() for block in blocks if len(block) > 1
}
rank_tuple_to_cpu_pg: dict[tuple[int, ...], "dist.ProcessGroup"] = {}
for rank_tuple in sorted(axis_tuples | instance_tuples):
member = self.global_rank in rank_tuple
if device.type == "xpu":
if rank_tuple in axis_tuples:
continue # the device group doubles as the cpu group below
cpu_pg = dist.new_group(ranks=list(rank_tuple), **timeout_kwargs)
else:
cpu_pg = dist.new_group(
ranks=list(rank_tuple) if member else [self.global_rank],
backend="gloo", use_local_synchronization=True, **timeout_kwargs,
)
if member:
rank_tuple_to_cpu_pg[rank_tuple] = cpu_pg
# One device subgroup per distinct rank tuple over every mesh axis —
# ``world_parallel_groups`` is the sorted union, so every rank
# iterates it identically; ``new_group`` is collective and
# tag-ordered (see the field comment). A tuple shared by two axes'
# groups (degenerate meshes) maps to one subgroup.
rank_tuple_to_pg: dict[tuple[int, ...], "dist.ProcessGroup"] = {}
for rank_tuple in self.world_parallel_groups:
rank_tuple_to_pg[rank_tuple] = dist.new_group(
ranks=list(rank_tuple), **timeout_kwargs
)
if device.type == "xpu":
rank_tuple_to_cpu_pg[rank_tuple] = rank_tuple_to_pg[rank_tuple]
seen: set[int] = set()
for comm_group in (group for per_node in self.axis_groups.values() for group in per_node.values()):
if id(comm_group) in seen:
continue
seen.add(id(comm_group))
if comm_group.world_size == 1:
comm_group.initialized = True
continue
comm_group.device_group = rank_tuple_to_pg[tuple(comm_group.group_members)]
comm_group.cpu_group = rank_tuple_to_cpu_pg[tuple(comm_group.group_members)]
comm_group.initialized = True
# Collective rendezvous within the group. Members visit shared
# groups in the same relative order: GlobalParallelConfig fills
# every worker's dicts in one global iteration order.
comm_group.init_all_reduce(self.tp_allreduce, self.tp_allreduce_max_kb)
for node, comm_group in self.node_to_instance_comm.items():
if comm_group.world_size > 1:
cpu_pg = rank_tuple_to_cpu_pg.get(tuple(comm_group.group_members))
if cpu_pg is None:
raise ValueError(
f"node {node!r}: instance ranks {comm_group.group_members} are not "
f"among its instance groups "
f"{sorted(self.node_to_instance_groups.get(node, ()))}"
)
comm_group.cpu_group = cpu_pg
comm_group.initialized = True
def _all_groups(self) -> list[CommGroup]:
"""Every axis group, then every instance group."""
return [
*(group for per_node in self.axis_groups.values() for group in per_node.values()),
*self.node_to_instance_comm.values(),
]
def get_joint_group_for_node(self, node: str) -> JointGroups:
if node not in self.node_to_joint_group:
axes = {
f"{axis}_group": self.axis_groups[axis].setdefault(node, CommGroup.trivial())
for axis in PARALLEL_AXES
}
instance = self.node_to_instance_comm.get(node)
if instance is None:
if any(group.world_size > 1 for group in axes.values()):
# every rank would lead it, and every vote on it would pass
raise ValueError(f"node {node!r} has a parallel group but no lockstep instance group")
instance = CommGroup.trivial()
self.node_to_joint_group[node] = JointGroups(**axes, instance=instance)
return self.node_to_joint_group[node]
def get_instance_rank_for_node(self, node: str) -> int:
return self.get_joint_group_for_node(node).rank
def get_instance_world_size_for_node(self, node: str) -> int:
return self.get_joint_group_for_node(node).world_size
def all_in_same_group(self, nodes: list[str]) -> bool:
"""Whether every node shares one group on every mesh axis.
Per axis, and only where someone parallelizes: unregistered reads
as the single-rank group, so a TP-only run would otherwise be rejected
for "differing" on SP. By membership, not identity — the lazy getters
mint a fresh group per node (and would cache one for a remote node).
"""
for per_node in self.axis_groups.values():
members = [
tuple(group.group_members) if group is not None else (0,)
for group in (per_node.get(node) for node in nodes)
]
if all(len(m) == 1 for m in members):
continue
if len(set(members)) != 1:
return False
return True
def all_have_compatible_parallel_shape(self, nodes: set[str]) -> bool:
"""Whether every replica uses the same size on every mesh axis."""
shapes: set[tuple[int, ...]] = set()
for node in nodes:
global_shapes = self.node_to_parallel_shapes.get(node)
if global_shapes:
shapes.update(global_shapes)
continue
shapes.add(tuple(
group.world_size if (group := self.axis_groups[axis].get(node)) is not None else 1
for axis in PARALLEL_AXES
))
return len(shapes) <= 1
def resource_needs_remote_transfer(
self, nodes: set[str], local_nodes: set[str],
) -> bool:
"""Whether a logical resource spans more than one worker instance."""
instance_groups: set[tuple[int, ...]] = set()
topology_known = True
for node in nodes:
groups = self.node_to_instance_groups.get(node)
if not groups:
topology_known = False
break
instance_groups.update(groups)
if topology_known:
return len(instance_groups) > 1
# Preserve sensible behavior for hand-built/test configs that lack the
# global maps: another named consumer necessarily needs a transfer.
return bool(nodes - local_nodes)
def barrier_all(self) -> None:
"""Global barrier across every worker process in the run.
No-op when ``any_parallelism`` is False (no NCCL world was
initialized in ``init_dist``). Otherwise calls ``dist.barrier()``
on the default global process group, syncing participating and
non-participating workers alike. Used where every rank must be
ready before proceeding — e.g. between CUDA-graph warmup and the
worker's main loop, so an instance leader can't send a
``ScheduleTPNode`` to a follower that's still inside
``engine.warmup``.
"""
if not self.any_parallelism:
return
_barrier()
def restore_default_timeout(self) -> None:
"""Give every group this rank holds torch's default timeout back."""
# the long timeout is for load and warmup; in serving a wedged rank
# must abort the others, not leave them waiting for hours
if not self.any_parallelism or not resolve_dist_timeout(self.dist_timeout_s):
return
groups = [dist.group.WORLD]
for comm_group in self._all_groups():
groups += [comm_group.device_group, comm_group.cpu_group]
for pg in {id(g): g for g in groups if g is not None}.values():
nccl = dist.get_backend(pg) == dist.Backend.NCCL
_set_pg_timeout(default_pg_nccl_timeout if nccl else default_pg_timeout, pg)
class GlobalParallelConfig:
def __init__(
# leaving type annotation as Any due to circular import
self, worker_graphs: dict[str, Any],
worker_ids: list[str],
dist_timeout_s: float | None = None,
tp_allreduce: str | None = None,
tp_allreduce_max_kb: int | None = None,
):
self.num_workers = len(worker_ids)
any_parallelism = any(wg.instance_size > 1 for wg in worker_graphs.values())
node_to_parallel_shapes: dict[str, set[tuple[int, ...]]] = {}
node_to_instance_groups: dict[str, set[tuple[int, ...]]] = {}
for wg in worker_graphs.values():
shape = tuple(wg.axis_sizes[axis] for axis in PARALLEL_AXES)
for node in wg.section.get_nodes():
node_to_parallel_shapes.setdefault(node, set()).add(shape)
node_to_instance_groups.setdefault(node, set()).update(
tuple(group) for group in wg.replica_ranks
)
frozen_parallel_shapes = {
node: frozenset(shapes)
for node, shapes in node_to_parallel_shapes.items()
}
frozen_instance_groups = {
node: frozenset(groups)
for node, groups in node_to_instance_groups.items()
}
# The union over every axis, sorted — the per-rank ``new_group``
# schedule (see the ``world_parallel_groups`` field comment).
world_parallel_groups: list[tuple[int, ...]] = sorted({
tuple(rank_group)
for wg in worker_graphs.values()
for rank_groups in wg._axis_ranks.values()
for rank_group in rank_groups
if len(rank_group) > 1
})
self.per_worker_config: dict[str, WorkerParallelGroups] = {
wid: WorkerParallelGroups(
global_rank=i, num_workers=self.num_workers,
any_parallelism=any_parallelism,
world_parallel_groups=world_parallel_groups,
dist_timeout_s=dist_timeout_s,
tp_allreduce=tp_allreduce,
tp_allreduce_max_kb=tp_allreduce_max_kb,
node_to_parallel_shapes=frozen_parallel_shapes,
node_to_instance_groups=frozen_instance_groups,
) for i, wid in enumerate(worker_ids)
}
# (axis, global rank, (group ranks...)) -> comm group; axis
# "instance" for the whole instance
comm_groups: dict[tuple[str, int, tuple], CommGroup] = {}
for wg in worker_graphs.values():
for axis, rank_groups in wg._axis_ranks.items():
for rank_group in rank_groups:
for i, rank in enumerate(rank_group):
key = (axis, rank, tuple(rank_group))
if key not in comm_groups:
comm_groups[key] = CommGroup(
my_global_rank=rank, my_group_rank=i, group_members=rank_group,
)
for node in wg.section.get_nodes():
self.per_worker_config[worker_ids[rank]].add(node, comm_groups[key], axis)
# first wins: ``add`` refused a node whose multi-rank instances
# differ on a rank, since they differ there on some axis (size-1
# groups count). A worker graph with one-rank instances registers
# no groups.
for block in wg._instance_ranks:
for i, rank in enumerate(block):
key = ("instance", rank, tuple(block))
if key not in comm_groups:
comm_groups[key] = CommGroup(my_global_rank=rank, my_group_rank=i, group_members=block)
for node in wg.section.get_nodes():
self.per_worker_config[worker_ids[rank]].node_to_instance_comm.setdefault(
node, comm_groups[key],
)